stellar stat parses output related configuration items

This commit is contained in:
luwenpeng
2024-09-02 16:03:08 +08:00
parent 9069dceae7
commit beb7d2f0ca
9 changed files with 228 additions and 294 deletions

View File

@@ -237,13 +237,14 @@ const char *name[] = {
struct stellar_stat
{
uint16_t nr_thread;
char output_file[2048];
struct stellar_stat_config cfg;
struct fieldstat_easy *fs;
uint64_t last_merge_stat_ts;
uint64_t last_output_stat_ts;
int flag[MAX_THREAD_NUM]; // IS_FREE or IS_BUSY
struct thread_stat thr_stat[MAX_THREAD_NUM];
uint64_t stat_idx[STAT_TYPE_MAX];
uint64_t stat_val[STAT_TYPE_MAX];
};
@@ -254,177 +255,177 @@ uint64_t get_stat_value_by_idx(const struct thread_stat *thr_stat, size_t idx)
{
// device packet
case STAT_TYPE_PKTS_RX:
return thr_stat->packet_io->pkts_rx;
return thr_stat->pkt_io->pkts_rx;
case STAT_TYPE_BYTES_RX:
return thr_stat->packet_io->bytes_rx;
return thr_stat->pkt_io->bytes_rx;
case STAT_TYPE_PKTS_TX:
return thr_stat->packet_io->pkts_tx;
return thr_stat->pkt_io->pkts_tx;
case STAT_TYPE_BYTES_TX:
return thr_stat->packet_io->bytes_tx;
return thr_stat->pkt_io->bytes_tx;
// keep-alive packet
case STAT_TYPE_KEEP_ALIVE_PKTS:
return thr_stat->packet_io->keep_alive_pkts;
return thr_stat->pkt_io->keep_alive_pkts;
case STAT_TYPE_KEEP_ALIVE_BYTES:
return thr_stat->packet_io->keep_alive_bytes;
return thr_stat->pkt_io->keep_alive_bytes;
// raw packet
case STAT_TYPE_RAW_PKTS_RX:
return thr_stat->packet_io->raw_pkts_rx;
return thr_stat->pkt_io->raw_pkts_rx;
case STAT_TYPE_RAW_BYTES_RX:
return thr_stat->packet_io->raw_bytes_rx;
return thr_stat->pkt_io->raw_bytes_rx;
case STAT_TYPE_RAW_PKTS_TX:
return thr_stat->packet_io->raw_pkts_tx;
return thr_stat->pkt_io->raw_pkts_tx;
case STAT_TYPE_RAW_BYTES_TX:
return thr_stat->packet_io->raw_bytes_tx;
return thr_stat->pkt_io->raw_bytes_tx;
// drop packet
case STAT_TYPE_PKTS_DROPPED:
return thr_stat->packet_io->pkts_dropped;
return thr_stat->pkt_io->pkts_dropped;
case STAT_TYPE_BYTES_DROPPED:
return thr_stat->packet_io->bytes_dropped;
return thr_stat->pkt_io->bytes_dropped;
// inject packet
case STAT_TYPE_PKTS_INJECTED:
return thr_stat->packet_io->pkts_injected;
return thr_stat->pkt_io->pkts_injected;
case STAT_TYPE_BYTES_INJECTED:
return thr_stat->packet_io->bytes_injected;
return thr_stat->pkt_io->bytes_injected;
// ctrl packet
case STAT_TYPE_CTRL_PKTS_RX:
return thr_stat->packet_io->ctrl_pkts_rx;
return thr_stat->pkt_io->ctrl_pkts_rx;
case STAT_TYPE_CTRL_BYTES_RX:
return thr_stat->packet_io->ctrl_bytes_rx;
return thr_stat->pkt_io->ctrl_bytes_rx;
case STAT_TYPE_CTRL_PKTS_TX:
return thr_stat->packet_io->ctrl_pkts_tx;
return thr_stat->pkt_io->ctrl_pkts_tx;
case STAT_TYPE_CTRL_BYTES_TX:
return thr_stat->packet_io->ctrl_bytes_tx;
return thr_stat->pkt_io->ctrl_bytes_tx;
// ipv4 reassembly
case STAT_TYPE_IP4_DEFRAGS_EXPECTED:
return thr_stat->ip_reassembly->ip4_defrags_expected;
return thr_stat->ip_reass->ip4_defrags_expected;
case STAT_TYPE_IP4_DEFRAGS_SUCCEED:
return thr_stat->ip_reassembly->ip4_defrags_succeed;
return thr_stat->ip_reass->ip4_defrags_succeed;
case STAT_TYPE_IP4_DEFRAGS_FAILED_TIMEOUT:
return thr_stat->ip_reassembly->ip4_defrags_failed_timeout;
return thr_stat->ip_reass->ip4_defrags_failed_timeout;
case STAT_TYPE_IP4_DEFRAGS_FAILED_INVALID_LENGTH:
return thr_stat->ip_reassembly->ip4_defrags_failed_invalid_length;
return thr_stat->ip_reass->ip4_defrags_failed_invalid_length;
case STAT_TYPE_IP4_DEFRAGS_FAILED_OVERLAP:
return thr_stat->ip_reassembly->ip4_defrags_failed_overlap;
return thr_stat->ip_reass->ip4_defrags_failed_overlap;
case STAT_TYPE_IP4_DEFRAGS_FAILED_TOO_MANY_FRAG:
return thr_stat->ip_reassembly->ip4_defrags_failed_too_many_frag;
return thr_stat->ip_reass->ip4_defrags_failed_too_many_frag;
case STAT_TYPE_IP4_FRAGS:
return thr_stat->ip_reassembly->ip4_frags;
return thr_stat->ip_reass->ip4_frags;
case STAT_TYPE_IP4_FRAGS_FREED:
return thr_stat->ip_reassembly->ip4_frags_freed;
return thr_stat->ip_reass->ip4_frags_freed;
case STAT_TYPE_IP4_FRAGS_BUFFERED:
return thr_stat->ip_reassembly->ip4_frags_buffered;
return thr_stat->ip_reass->ip4_frags_buffered;
case STAT_TYPE_IP4_FRAGS_BYPASS_NO_BUFFER:
return thr_stat->ip_reassembly->ip4_frags_bypass_no_buffer;
return thr_stat->ip_reass->ip4_frags_bypass_no_buffer;
case STAT_TYPE_IP4_FRAGS_BYPASS_DUP_FIST_FRAG:
return thr_stat->ip_reassembly->ip4_frags_bypass_dup_fist_frag;
return thr_stat->ip_reass->ip4_frags_bypass_dup_fist_frag;
case STAT_TYPE_IP4_FRAGS_BYPASS_DUP_LAST_FRAG:
return thr_stat->ip_reassembly->ip4_frags_bypass_dup_last_frag;
return thr_stat->ip_reass->ip4_frags_bypass_dup_last_frag;
// ipv6 reassembly
case STAT_TYPE_IP6_DEFRAGS_EXPECTED:
return thr_stat->ip_reassembly->ip6_defrags_expected;
return thr_stat->ip_reass->ip6_defrags_expected;
case STAT_TYPE_IP6_DEFRAGS_SUCCEED:
return thr_stat->ip_reassembly->ip6_defrags_succeed;
return thr_stat->ip_reass->ip6_defrags_succeed;
case STAT_TYPE_IP6_DEFRAGS_FAILED_TIMEOUT:
return thr_stat->ip_reassembly->ip6_defrags_failed_timeout;
return thr_stat->ip_reass->ip6_defrags_failed_timeout;
case STAT_TYPE_IP6_DEFRAGS_FAILED_INVALID_LENGTH:
return thr_stat->ip_reassembly->ip6_defrags_failed_invalid_length;
return thr_stat->ip_reass->ip6_defrags_failed_invalid_length;
case STAT_TYPE_IP6_DEFRAGS_FAILED_OVERLAP:
return thr_stat->ip_reassembly->ip6_defrags_failed_overlap;
return thr_stat->ip_reass->ip6_defrags_failed_overlap;
case STAT_TYPE_IP6_DEFRAGS_FAILED_TOO_MANY_FRAG:
return thr_stat->ip_reassembly->ip6_defrags_failed_too_many_frag;
return thr_stat->ip_reass->ip6_defrags_failed_too_many_frag;
case STAT_TYPE_IP6_FRAGS:
return thr_stat->ip_reassembly->ip6_frags;
return thr_stat->ip_reass->ip6_frags;
case STAT_TYPE_IP6_FRAGS_FREED:
return thr_stat->ip_reassembly->ip6_frags_freed;
return thr_stat->ip_reass->ip6_frags_freed;
case STAT_TYPE_IP6_FRAGS_BUFFERED:
return thr_stat->ip_reassembly->ip6_frags_buffered;
return thr_stat->ip_reass->ip6_frags_buffered;
case STAT_TYPE_IP6_FRAGS_BYPASS_NO_BUFFER:
return thr_stat->ip_reassembly->ip6_frags_bypass_no_buffer;
return thr_stat->ip_reass->ip6_frags_bypass_no_buffer;
case STAT_TYPE_IP6_FRAGS_BYPASS_DUP_FIST_FRAG:
return thr_stat->ip_reassembly->ip6_frags_bypass_dup_fist_frag;
return thr_stat->ip_reass->ip6_frags_bypass_dup_fist_frag;
case STAT_TYPE_IP6_FRAGS_BYPASS_DUP_LAST_FRAG:
return thr_stat->ip_reassembly->ip6_frags_bypass_dup_last_frag;
return thr_stat->ip_reass->ip6_frags_bypass_dup_last_frag;
// TCP session
case STAT_TYPE_HISTORY_TCP_SESSIONS:
return thr_stat->session_mgr->history_tcp_sessions;
return thr_stat->sess_mgr->history_tcp_sessions;
case STAT_TYPE_TCP_SESS_USED:
return thr_stat->session_mgr->tcp_sess_used;
return thr_stat->sess_mgr->tcp_sess_used;
case STAT_TYPE_TCP_SESS_OPENING:
return thr_stat->session_mgr->tcp_sess_opening;
return thr_stat->sess_mgr->tcp_sess_opening;
case STAT_TYPE_TCP_SESS_ACTIVE:
return thr_stat->session_mgr->tcp_sess_active;
return thr_stat->sess_mgr->tcp_sess_active;
case STAT_TYPE_TCP_SESS_CLOSING:
return thr_stat->session_mgr->tcp_sess_closing;
return thr_stat->sess_mgr->tcp_sess_closing;
case STAT_TYPE_TCP_SESS_DISCARD:
return thr_stat->session_mgr->tcp_sess_discard;
return thr_stat->sess_mgr->tcp_sess_discard;
case STAT_TYPE_TCP_SESS_CLOSED:
return thr_stat->session_mgr->tcp_sess_closed;
return thr_stat->sess_mgr->tcp_sess_closed;
// UDP session
case STAT_TYPE_HISTORY_UDP_SESSIONS:
return thr_stat->session_mgr->history_udp_sessions;
return thr_stat->sess_mgr->history_udp_sessions;
case STAT_TYPE_UDP_SESS_USED:
return thr_stat->session_mgr->udp_sess_used;
return thr_stat->sess_mgr->udp_sess_used;
case STAT_TYPE_UDP_SESS_OPENING:
return thr_stat->session_mgr->udp_sess_opening;
return thr_stat->sess_mgr->udp_sess_opening;
case STAT_TYPE_UDP_SESS_ACTIVE:
return thr_stat->session_mgr->udp_sess_active;
return thr_stat->sess_mgr->udp_sess_active;
case STAT_TYPE_UDP_SESS_CLOSING:
return thr_stat->session_mgr->udp_sess_closing;
return thr_stat->sess_mgr->udp_sess_closing;
case STAT_TYPE_UDP_SESS_DISCARD:
return thr_stat->session_mgr->udp_sess_discard;
return thr_stat->sess_mgr->udp_sess_discard;
case STAT_TYPE_UDP_SESS_CLOSED:
return thr_stat->session_mgr->udp_sess_closed;
return thr_stat->sess_mgr->udp_sess_closed;
// Evicted session
case STAT_TYPE_TCP_SESS_EVICTED:
return thr_stat->session_mgr->tcp_sess_evicted;
return thr_stat->sess_mgr->tcp_sess_evicted;
case STAT_TYPE_UDP_SESS_EVICTED:
return thr_stat->session_mgr->udp_sess_evicted;
return thr_stat->sess_mgr->udp_sess_evicted;
// Packet
case STAT_TYPE_UDP_PKTS_BYPASS_TABLE_FULL:
return thr_stat->session_mgr->udp_pkts_bypass_table_full;
return thr_stat->sess_mgr->udp_pkts_bypass_table_full;
case STAT_TYPE_TCP_PKTS_BYPASS_TABLE_FULL:
return thr_stat->session_mgr->tcp_pkts_bypass_table_full;
return thr_stat->sess_mgr->tcp_pkts_bypass_table_full;
case STAT_TYPE_TCP_PKTS_BYPASS_SESSION_NOT_FOUND:
return thr_stat->session_mgr->tcp_pkts_bypass_session_not_found;
return thr_stat->sess_mgr->tcp_pkts_bypass_session_not_found;
case STAT_TYPE_TCP_PKTS_BYPASS_DUPLICATED:
return thr_stat->session_mgr->tcp_pkts_bypass_duplicated;
return thr_stat->sess_mgr->tcp_pkts_bypass_duplicated;
case STAT_TYPE_UDP_PKTS_BYPASS_DUPLICATED:
return thr_stat->session_mgr->udp_pkts_bypass_duplicated;
return thr_stat->sess_mgr->udp_pkts_bypass_duplicated;
case STAT_TYPE_UDP_PKTS_BYPASS_SESSION_EVICTED:
return thr_stat->session_mgr->udp_pkts_bypass_session_evicted;
return thr_stat->sess_mgr->udp_pkts_bypass_session_evicted;
// TCP segments
case STAT_TYPE_TCP_SEGS_INPUT:
return thr_stat->session_mgr->tcp_segs_input;
return thr_stat->sess_mgr->tcp_segs_input;
case STAT_TYPE_TCP_SEGS_CONSUMED:
return thr_stat->session_mgr->tcp_segs_consumed;
return thr_stat->sess_mgr->tcp_segs_consumed;
case STAT_TYPE_TCP_SEGS_TIMEOUT:
return thr_stat->session_mgr->tcp_segs_timeout;
return thr_stat->sess_mgr->tcp_segs_timeout;
case STAT_TYPE_TCP_SEGS_RETRANSMITED:
return thr_stat->session_mgr->tcp_segs_retransmited;
return thr_stat->sess_mgr->tcp_segs_retransmited;
case STAT_TYPE_TCP_SEGS_OVERLAPPED:
return thr_stat->session_mgr->tcp_segs_overlapped;
return thr_stat->sess_mgr->tcp_segs_overlapped;
case STAT_TYPE_TCP_SEGS_OMITTED_TOO_MANY:
return thr_stat->session_mgr->tcp_segs_omitted_too_many;
return thr_stat->sess_mgr->tcp_segs_omitted_too_many;
case STAT_TYPE_TCP_SEGS_INORDER:
return thr_stat->session_mgr->tcp_segs_inorder;
return thr_stat->sess_mgr->tcp_segs_inorder;
case STAT_TYPE_TCP_SEGS_REORDERED:
return thr_stat->session_mgr->tcp_segs_reordered;
return thr_stat->sess_mgr->tcp_segs_reordered;
case STAT_TYPE_TCP_SEGS_BUFFERED:
return thr_stat->session_mgr->tcp_segs_buffered;
return thr_stat->sess_mgr->tcp_segs_buffered;
case STAT_TYPE_TCP_SEGS_FREED:
return thr_stat->session_mgr->tcp_segs_freed;
return thr_stat->sess_mgr->tcp_segs_freed;
default:
assert(0);
@@ -432,10 +433,55 @@ uint64_t get_stat_value_by_idx(const struct thread_stat *thr_stat, size_t idx)
}
}
struct stellar_stat_config *stellar_stat_config_new(const char *toml_file)
{
if (toml_file == NULL)
{
return NULL;
}
struct stellar_stat_config *cfg = (struct stellar_stat_config *)calloc(1, sizeof(struct stellar_stat_config));
if (cfg == NULL)
{
return NULL;
}
int ret = 0;
ret += load_and_validate_toml_integer_config(toml_file, "packet_io.nr_worker_thread", (uint64_t *)&cfg->nr_worker_thread, 1, MAX_THREAD_NUM);
ret += load_and_validate_toml_integer_config(toml_file, "stat.merge_interval_ms", (uint64_t *)&cfg->merge_interval_ms, 0, 60000);
ret += load_and_validate_toml_integer_config(toml_file, "stat.output_interval_ms", (uint64_t *)&cfg->output_interval_ms, 0, 60000);
if (ret != 0)
{
stellar_stat_config_free(cfg);
return NULL;
}
return cfg;
}
void stellar_stat_config_free(struct stellar_stat_config *cfg)
{
if (cfg)
{
free(cfg);
cfg = NULL;
}
}
void stellar_stat_config_print(const struct stellar_stat_config *cfg)
{
if (cfg)
{
STAT_LOG_INFO("stat.merge_interval_ms : %lu", cfg->merge_interval_ms);
STAT_LOG_INFO("stat.output_interval_ms : %lu", cfg->output_interval_ms);
}
}
// python3 -m pip install prettytable
// python3 -m pip install jinja2
// /opt/MESA/bin/fieldstat_exporter.py local -j log/stellar_fs4.json -e -l --clear-screen
struct stellar_stat *stellar_stat_new(uint16_t nr_thread)
struct stellar_stat *stellar_stat_new(const struct stellar_stat_config *cfg, uint64_t now_ms)
{
struct stellar_stat *stat = (struct stellar_stat *)calloc(1, sizeof(struct stellar_stat));
if (stat == NULL)
@@ -443,7 +489,7 @@ struct stellar_stat *stellar_stat_new(uint16_t nr_thread)
return NULL;
}
snprintf(stat->output_file, sizeof(stat->output_file), "./log/stellar_fs4.json");
memcpy(&stat->cfg, cfg, sizeof(struct stellar_stat_config));
stat->fs = fieldstat_easy_new(1, "stellar", NULL, 0);
if (stat->fs == NULL)
@@ -452,7 +498,6 @@ struct stellar_stat *stellar_stat_new(uint16_t nr_thread)
goto error_out;
}
stat->nr_thread = nr_thread;
for (int i = 0; i < MAX_THREAD_NUM; i++)
{
stat->flag[i] = IS_FREE;
@@ -463,6 +508,9 @@ struct stellar_stat *stellar_stat_new(uint16_t nr_thread)
stat->stat_idx[i] = fieldstat_easy_register_counter(stat->fs, name[i]);
}
stat->last_merge_stat_ts = now_ms;
stat->last_output_stat_ts = now_ms;
return stat;
error_out:
@@ -484,9 +532,15 @@ void stellar_stat_free(struct stellar_stat *stat)
}
}
void stellar_stat_output(struct stellar_stat *stat)
void stellar_stat_output(struct stellar_stat *stat, uint64_t now_ms)
{
for (uint16_t i = 0; i < stat->nr_thread; i++)
if (now_ms - stat->last_output_stat_ts < stat->cfg.output_interval_ms)
{
return;
}
stat->last_output_stat_ts = now_ms;
for (uint16_t i = 0; i < stat->cfg.nr_worker_thread; i++)
{
if (ATOMIC_READ(&(stat->flag[i])) == IS_BUSY)
{
@@ -510,10 +564,10 @@ void stellar_stat_output(struct stellar_stat *stat)
fieldstat_easy_output(stat->fs, &buff, &len);
if (buff)
{
FILE *fp = fopen(stat->output_file, "w+");
FILE *fp = fopen("./log/stellar_fs4.json", "w+");
if (fp == NULL)
{
STAT_LOG_ERROR("failed to open file: %s, %s", stat->output_file, strerror(errno));
STAT_LOG_ERROR("failed to open file: ./log/stellar_fs4.json, %s", strerror(errno));
}
else
{
@@ -530,12 +584,18 @@ void stellar_stat_output(struct stellar_stat *stat)
}
}
void stellar_stat_merge(struct stellar_stat *stat, const struct thread_stat *thr_stat, uint16_t thr_idx)
void stellar_stat_merge(struct stellar_stat *stat, const struct thread_stat *thr_stat, uint16_t thr_idx, uint64_t now_ms)
{
if (now_ms - stat->last_merge_stat_ts < stat->cfg.merge_interval_ms)
{
return;
}
if (ATOMIC_READ(&(stat->flag[thr_idx])) == IS_FREE)
{
memcpy(&stat->thr_stat[thr_idx], thr_stat, sizeof(struct thread_stat));
ATOMIC_SET(&(stat->flag[thr_idx]), IS_BUSY);
stat->last_merge_stat_ts = now_ms;
}
}