This repository has been archived on 2025-09-14. You can view files and clone it, but cannot push or open issues or pull requests.
Files
stellar-stellar/test/monitor/gtest_topk.cpp
2024-11-07 18:30:58 +08:00

535 lines
19 KiB
C++

#include <assert.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stddef.h>
#include <getopt.h>
#include <sys/time.h>
#include <gtest/gtest.h>
#include <arpa/inet.h>
// #include <fieldstat.h>
#include <fieldstat/fieldstat_exporter.h>
#define MAX_TOPK_CELL_NUM 100
#define MAX_TOPK_METRIC_NUM 255
#define SHOW_TOPK_RESULT_NUM 10
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define TOPK_DIMENSION_NUM 8
static const char *g_topk_dimension[TOPK_DIMENSION_NUM] = {
"top_client_ip",
"top_server_ip",
"top_internal_ip",
"top_external_ip",
"top_server_fqdn",
"top_server_domain",
"top_client_country",
"top_server_country",
};
#define TOPK_RANK_BY_NUM 3
static const char *g_topk_rank_by[TOPK_RANK_BY_NUM] = {
"sessions",
"packets",
"bytes",
};
#define TOPK_METRIC_NUM 3
/* index corresponding to g_topk_rank_by[] */
static const char *g_metric_name[TOPK_METRIC_NUM] = {
"sessions",
"pkts",
"bytes",
};
struct topk_t
{
char name[128];
long long value;
};
static int g_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {}; // [dimension][rank_by]
static int g_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]
static struct fieldstat *g_fs4_ins = NULL;
/* the biggest value index is 0 */
static int qsort_topk_value_cmp_cb(const void *a, const void *b)
{
struct topk_t *la = (struct topk_t *)a;
struct topk_t *lb = (struct topk_t *)b;
return (int)(lb->value - la->value);
}
// lijia temp add for test
static void show_topk_stat(const char *dimension, const char *rank_by, struct topk_t *topk_result, size_t n_result, size_t topk)
{
if (n_result)
{
printf("### top '%s' rank_by '%s' result %zu: \n", dimension, rank_by, topk);
qsort((void *)topk_result, n_result, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
int show_num = MIN(topk, n_result);
for (int i = 0; i < show_num; i++)
{
printf("%2d) %16s\t%8lld\n", i + 1, topk_result[i].name, topk_result[i].value);
}
}
}
static int get_topk_cube_id(struct fieldstat *fs4_ins, const char *dimension, const char *rank_by)
{
struct field find_cube_tmp[2] = {};
find_cube_tmp[0].key = "name";
find_cube_tmp[0].type = FIELD_VALUE_CSTRING;
find_cube_tmp[0].value_str = dimension;
find_cube_tmp[1].key = "rank_by";
find_cube_tmp[1].type = FIELD_VALUE_CSTRING;
find_cube_tmp[1].value_str = rank_by;
return fieldstat_find_cube(fs4_ins, find_cube_tmp, 2);
}
/*
dimensions: client_ip, server_ip, internal_ip, external_ip, server_fqdn, server_domain, client_country, server_country
rank_by: sessions, bytes, packets
metric: sessions, bytes, packets
*/
static int get_topk_rank_by(struct fieldstat *fs4_ins, const char *dimension, const char *rank_by, const char *metric_name, struct topk_t *topk_array, size_t max_array_num)
{
int cube_id = get_topk_cube_id(fs4_ins, dimension, rank_by);
assert(cube_id >= 0);
int metric_id = fieldstat_cube_get_metric_id_by_name(fs4_ins, cube_id, metric_name);
assert(metric_id >= 0);
struct field_list *cell_dimensions_list = NULL;
size_t n_cell = 0;
fieldstat_cube_get_cells(fs4_ins, cube_id, &cell_dimensions_list, &n_cell);
if (NULL == cell_dimensions_list)
{
return 0;
}
size_t result_min = MIN(max_array_num, n_cell);
for (size_t ce = 0; ce < result_min; ce++)
{
strncpy(topk_array[ce].name, cell_dimensions_list[ce].field->value_str, sizeof(topk_array[ce].name));
fieldstat_counter_get(fs4_ins, cube_id, &cell_dimensions_list[ce], metric_id, &topk_array[ce].value);
}
fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
return result_min;
}
static void stm_topk_register_cube(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM])
{
struct field reg_cube_tmp[2] = {};
for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
{
for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
{
reg_cube_tmp[0].key = "name";
reg_cube_tmp[0].type = FIELD_VALUE_CSTRING;
reg_cube_tmp[0].value_str = g_topk_dimension[i];
reg_cube_tmp[1].key = "rank_by";
reg_cube_tmp[1].type = FIELD_VALUE_CSTRING;
reg_cube_tmp[1].value_str = g_topk_rank_by[j];
cube_id[i][j] = fieldstat_cube_create(fs4_ins, reg_cube_tmp, 2);
assert(cube_id[i][j] >= 0);
}
}
}
static void stm_topk_register_counter(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
{
for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
{
for (size_t k = 0; k < sizeof(g_metric_name) / sizeof(char *); k++)
{
counter_id[i][j][k] = fieldstat_register_counter(fs4_ins, cube_id[i][j], g_metric_name[k]);
assert(counter_id[i][j][k] >= 0);
}
}
}
}
static void stm_topk_set_cube_sampling(struct fieldstat *fs4_ins, enum sampling_mode mode, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
{
for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
{
fieldstat_cube_set_sampling(fs4_ins, cube_id[i][j], mode, MAX_TOPK_CELL_NUM, counter_id[i][j][j]);
}
}
}
static int stm_topk_client_ip_rankby_sessions(void)
{
struct field ip_tag = {};
ip_tag.key = "top_client_ip";
ip_tag.type = FIELD_VALUE_CSTRING;
unsigned int begin_ip_addr = 0x04030201;
struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
long long max_value = 0;
memset(benchmark_result, 0, sizeof(benchmark_result));
for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
{
benchmark_result[i].value = rand() % 10000;
if (benchmark_result[i].value > max_value)
{
max_value = benchmark_result[i].value;
}
unsigned int ip_addr_net = htonl(begin_ip_addr + i);
inet_ntop(AF_INET, &ip_addr_net, benchmark_result[i].name, sizeof(benchmark_result[i].name));
ip_tag.value_str = benchmark_result[i].name;
fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[0][0], g_topk_counter_id[0][0][0], &ip_tag, 1, benchmark_result[i].value);
}
/*** query and verify ***/
struct topk_t fs4_topk_result[MAX_TOPK_CELL_NUM] = {};
int num = get_topk_rank_by(g_fs4_ins, "top_client_ip", "sessions", "sessions", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
assert(num > 0);
printf("topk top_client_ip rankby sessions, max value:%lld\n", max_value);
show_topk_stat("top_client_ip", "sessions", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
assert(max_value == fs4_topk_result[0].value);
return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}
static int stm_topk_server_ip_rankby_packets(void)
{
struct field ip_tag = {};
ip_tag.key = "top_server_ip";
ip_tag.type = FIELD_VALUE_CSTRING;
unsigned int begin_ip_addr = 0x64640101;
long long max_value = 0;
struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
memset(benchmark_result, 0, sizeof(benchmark_result));
for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
{
benchmark_result[i].value = rand() % 10000;
if (benchmark_result[i].value > max_value)
{
max_value = benchmark_result[i].value;
}
unsigned int ip_addr_net = htonl(begin_ip_addr + i);
inet_ntop(AF_INET, &ip_addr_net, benchmark_result[i].name, sizeof(benchmark_result[i].name));
ip_tag.value_str = benchmark_result[i].name;
fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[1][1], g_topk_counter_id[1][1][1], &ip_tag, 1, benchmark_result[i].value);
}
/*** query and verify ***/
struct topk_t fs4_topk_result[MAX_TOPK_CELL_NUM] = {};
int num = get_topk_rank_by(g_fs4_ins, "top_server_ip", "packets", "pkts", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
assert(num > 0);
printf("topk top_server_ip rankby packets, max value:%lld\n", max_value);
show_topk_stat("top_server_ip", "packets", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
assert(max_value == fs4_topk_result[0].value);
return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}
static int stm_topk_server_domain_rankby_bytes(void)
{
struct field domain_tag = {};
domain_tag.key = "top_server_domain";
domain_tag.type = FIELD_VALUE_CSTRING;
struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
memset(benchmark_result, 0, sizeof(benchmark_result));
for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
{
snprintf(benchmark_result[i].name, sizeof(benchmark_result[i].name), "www.abcd.com");
}
for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
{
benchmark_result[i].value = rand() % 10000;
benchmark_result[i].name[4] = rand() % 26 + 'a';
benchmark_result[i].name[5] = rand() % 26 + 'a';
benchmark_result[i].name[6] = rand() % 26 + 'a';
benchmark_result[i].name[7] = rand() % 26 + 'a';
domain_tag.value_str = benchmark_result[i].name;
fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[5][2], g_topk_counter_id[5][2][2], &domain_tag, 1, benchmark_result[i].value);
}
/*** query and verify ***/
struct topk_t fs4_topk_result[SHOW_TOPK_RESULT_NUM] = {};
int num = get_topk_rank_by(g_fs4_ins, "top_server_domain", "bytes", "bytes", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
assert(num > 0);
show_topk_stat("top_server_domain", "bytes", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}
class MONITOR_TOPK : public testing::Test
{
protected:
static void SetUpTestCase()
{
srand(time(NULL));
g_fs4_ins = fieldstat_new();
assert(g_fs4_ins != NULL);
stm_topk_register_cube(g_fs4_ins, g_topk_cubeid);
stm_topk_register_counter(g_fs4_ins, g_topk_cubeid, g_topk_counter_id);
stm_topk_set_cube_sampling(g_fs4_ins, SAMPLING_MODE_TOPK, g_topk_cubeid, g_topk_counter_id);
}
static void TearDownTestCase()
{
fieldstat_reset(g_fs4_ins);
fieldstat_free(g_fs4_ins);
}
};
static void generate_random_counter(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
struct field tag = {};
tag.type = FIELD_VALUE_CSTRING;
tag.value_str = "random";
for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
{
for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
{
for (size_t k = 0; k < sizeof(g_metric_name) / sizeof(char *); k++)
{
assert(g_topk_counter_id[i][j][k] >= 0);
tag.key = g_topk_dimension[i];
fieldstat_counter_incrby(fs4_ins, cube_id[i][j], counter_id[i][j][k], &tag, 1, random());
}
}
}
}
char **show_topk_get_metrics_name(struct fieldstat *show_fs4_ins, int *metric_array_num)
{
int *cube_id_array = NULL;
int n_cube = 0;
fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
if (NULL == cube_id_array)
{
return NULL;
}
struct field_list *cell_dimensions_list = NULL;
size_t n_cell = 0;
fieldstat_cube_get_cells(show_fs4_ins, cube_id_array[0], &cell_dimensions_list, &n_cell);
if (cell_dimensions_list == NULL)
{
return NULL;
}
/* get metric name of any valid cube_id */
int *metric_id_out;
size_t n_metric;
char **metric_array = NULL;
fieldstat_cell_get_metrics(show_fs4_ins, cube_id_array[0], cell_dimensions_list, &metric_id_out, &n_metric);
if (n_metric == 0)
{
fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
return NULL;
}
metric_array = (char **)malloc(sizeof(char *) * n_metric);
for (size_t j = 0; j < n_metric; j++)
{
metric_array[j] = strdup(fieldstat_metric_get_name(show_fs4_ins, cube_id_array[0], metric_id_out[j]));
}
fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
*metric_array_num = n_metric;
return metric_array;
}
TEST(MONITOR, show_all_valid_metrics)
{
struct fieldstat *show_fs4_ins = fieldstat_new();
int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {}; // [dimension][rank_by]
int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]
stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);
generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
int metric_array_num = 0;
char **metric_array = show_topk_get_metrics_name(show_fs4_ins, &metric_array_num);
EXPECT_TRUE(metric_array_num > 0);
EXPECT_TRUE(metric_array != NULL);
for (int i = 0; i < metric_array_num; i++)
{
printf("metric name: %s\n", metric_array[i]);
free(metric_array[i]);
}
free(metric_array);
fieldstat_free(show_fs4_ins);
}
static int charchar_array_exist(char **dimension_array, int dimension_array_num, const char *expect_name)
{
for (int i = 0; i < dimension_array_num && dimension_array[i] != NULL; i++)
{
if (0 == strcmp(dimension_array[i], expect_name))
{
return 1;
}
}
return 0;
}
char **show_topk_get_dimensions_name(struct fieldstat *show_fs4_ins, int *dimension_array_result)
{
int *cube_id_array = NULL;
int n_cube = 0;
fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
if (0 == n_cube)
{
return NULL;
}
char **dimension_array = NULL;
int dimension_array_num = 0;
struct field_list *cell_dimensions_list = NULL;
for (int i = 0; i < n_cube; i++)
{
size_t n_cell = 0;
fieldstat_cube_get_cells(show_fs4_ins, cube_id_array[i], &cell_dimensions_list, &n_cell);
if (NULL == cell_dimensions_list)
{
continue;
}
for (size_t j = 0; j < cell_dimensions_list->n_field; j++)
{
if (0 == charchar_array_exist(dimension_array, dimension_array_num, cell_dimensions_list->field[j].key))
{
dimension_array = (char **)realloc(dimension_array, sizeof(char *) * (dimension_array_num + 1));
dimension_array[dimension_array_num] = strdup(cell_dimensions_list->field[j].key);
dimension_array_num++;
}
}
fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
}
*dimension_array_result = dimension_array_num;
return dimension_array;
}
TEST(MONITOR, show_all_valid_dimensions)
{
struct fieldstat *show_fs4_ins = fieldstat_new();
int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {}; // [dimension][rank_by]
int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]
stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);
generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
int dimension_array_num = 0;
char **dimension_array = show_topk_get_dimensions_name(show_fs4_ins, &dimension_array_num);
EXPECT_TRUE(dimension_array_num > 0);
EXPECT_TRUE(dimension_array != NULL);
for (int i = 0; i < dimension_array_num; i++)
{
printf("dimension name: %s\n", dimension_array[i]);
free(dimension_array[i]);
}
free(dimension_array);
fieldstat_free(show_fs4_ins);
}
char **show_topk_get_rankby_name(struct fieldstat *show_fs4_ins, int *rankby_array_result)
{
int *cube_id_array = NULL;
int n_cube = 0;
fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
if (NULL == cube_id_array)
{
return NULL;
}
char **rankby_array = NULL;
int rankby_array_num = 0;
for (int i = 0; i < n_cube; i++)
{
struct field_list *flist = fieldstat_cube_get_dimension(show_fs4_ins, cube_id_array[i]);
if (flist == NULL)
{
continue;
}
for (size_t j = 0; j < flist->n_field; j++)
{
if (0 == strcmp(flist->field[j].key, "rank_by"))
{
if (0 == charchar_array_exist(rankby_array, rankby_array_num, flist->field[j].value_str))
{
rankby_array = (char **)realloc(rankby_array, sizeof(char *) * (rankby_array_num + 1));
rankby_array[rankby_array_num] = strdup(flist->field[j].value_str);
rankby_array_num++;
}
}
}
fieldstat_field_list_arr_free(flist, 1);
}
*rankby_array_result = rankby_array_num;
return rankby_array;
}
TEST(MONITOR, show_all_valid_rankby)
{
struct fieldstat *show_fs4_ins = fieldstat_new();
int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {}; // [dimension][rank_by]
int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]
stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);
generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
int rankby_array_num = 0;
char **rankby_array = show_topk_get_rankby_name(show_fs4_ins, &rankby_array_num);
EXPECT_TRUE(rankby_array_num > 0);
EXPECT_TRUE(rankby_array != NULL);
for (int i = 0; i < rankby_array_num; i++)
{
printf("rankby name: %s\n", rankby_array[i]);
free(rankby_array[i]);
}
free(rankby_array);
fieldstat_free(show_fs4_ins);
}
TEST_F(MONITOR_TOPK, show_client_ip_by_sessions)
{
EXPECT_EQ(0, stm_topk_client_ip_rankby_sessions());
}
TEST_F(MONITOR_TOPK, show_server_ip_by_packets)
{
EXPECT_EQ(0, stm_topk_server_ip_rankby_packets());
}
TEST_F(MONITOR_TOPK, show_server_domain_by_bytes)
{
EXPECT_EQ(0, stm_topk_server_domain_rankby_bytes());
}
int main(int argc, char **argv)
{
testing::InitGoogleTest(&argc, argv);
int ret = RUN_ALL_TESTS();
return ret;
}