gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-23378 TFE移除平台层的FieldStat2，为适配AArch64做准备

Browse Source
This commit is contained in:
luwenpeng
2024-11-13 14:09:03 +08:00
parent aa98cf915e
commit 3d99178958
18 changed files with 223 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/include/tfe_proxy.h
View File

@@ -1,5 +1,5 @@
#pragma once
#include <MESA/field_stat2.h>
#include <fieldstat/fieldstat_easy.h>
#include <event2/event.h>
struct tfe_proxy;
@@ -9,7 +9,7 @@ struct evdns_base* tfe_proxy_get_work_thread_dnsbase(unsigned int thread_id);
struct evhttp_connection* tfe_proxy_get_work_thread_evhttp(unsigned int thread_id);
struct event_base * tfe_proxy_get_gc_evbase(void);
screen_stat_handle_t tfe_proxy_get_fs_handle(void);
struct fieldstat_easy *tfe_proxy_get_fs_handle(void);
void * tfe_proxy_get_error_logger(void);
int tfe_proxy_ssl_add_trust_ca(const char* pem_file);

2
common/include/tfe_utils.h
View File

@@ -46,6 +46,7 @@
extern void * g_default_logger;
extern bool g_print_to_stderr;
extern thread_local unsigned int __currect_thread_id;
#define TFE_LOG_ERROR(handler, fmt, ...) \
do \
@@ -136,6 +137,7 @@ do { if(!(condition)) { TFE_LOG_ERROR(g_default_logger, fmt, ##__VA_ARGS__); abo
#define ATOMIC_ADD(x, y) __atomic_fetch_add(x,y,__ATOMIC_RELAXED)
#define ATOMIC_ZERO(x) __atomic_fetch_and(x,0,__ATOMIC_RELAXED)
#define ATOMIC_SET(x,y) __atomic_store_n(x,y,__ATOMIC_RELAXED)
#define ATOMIC_EXCHANGE(x,y) __atomic_exchange_n(x,y,__ATOMIC_RELAXED)
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif

200
common/src/tfe_future.cpp
View File

@@ -7,9 +7,8 @@
#include <tfe_future.h>
#include <tfe_utils.h>
#include <MESA/MESA_htable.h>
#include <MESA/field_stat2.h>
#include <MESA/MESA_prof_load.h>
#include <fieldstat/fieldstat_easy.h>
static const char* FP_HISTOGRAM_BINS="0.50,0.80,0.9,0.95,0.99";
@@ -17,15 +16,8 @@ struct future_promise_instance
{
int fsid_f_num;
long long f_num;
int no_stats;
int statsd_cycle;
int statsd_format;
MESA_htable_handle name_table;
char statsd_server_ip[256];
int statsd_server_port;
enum field_calc_algo favorite;
char histogram_bins[256];
screen_stat_handle_t fs_handle;
struct fieldstat_easy *fs_handle;
};
struct _future_promise_debug
@@ -57,84 +49,42 @@ struct promise
};
static struct future_promise_instance g_FP_instance;
static int g_is_FP_init=0;
void future_promise_library_init(const char* profile)
void future_promise_library_init(const char *profile)
{
if(g_is_FP_init==1)
if (g_is_FP_init == 1)
{
return;
}
int value=0;
memset(&g_FP_instance,0,sizeof(g_FP_instance));
g_FP_instance.favorite=FS_CALC_CURRENT;
strcpy(g_FP_instance.histogram_bins, FP_HISTOGRAM_BINS);
if(profile!=NULL)
{
MESA_load_profile_int_def(profile, "STAT", "no_stats", &(g_FP_instance.no_stats), 0);
MESA_load_profile_string_def(profile, "STAT", "statsd_server",
g_FP_instance.statsd_server_ip, sizeof(g_FP_instance.statsd_server_ip), "");
MESA_load_profile_int_def(profile, "STAT", "statsd_port", &(g_FP_instance.statsd_server_port), 0);
MESA_load_profile_int_def(profile, "STAT", "statsd_cycle", &(g_FP_instance.statsd_cycle), 2);
// FS_OUTPUT_STATSD=1, FS_OUTPUT_INFLUX_LINE=2
MESA_load_profile_int_def(profile, "STAT", "statsd_format", &(g_FP_instance.statsd_format), 1);
MESA_load_profile_string_def(profile, "STAT", "histogram_bins",
g_FP_instance.histogram_bins, sizeof(g_FP_instance.histogram_bins), FP_HISTOGRAM_BINS);
MESA_load_profile_int_def(profile, "STAT", "print_diff",
&value, 1);
if(value==0)
{
g_FP_instance.favorite=FS_CALC_CURRENT;
}
}
if(g_FP_instance.no_stats)
{
g_is_FP_init=1;
return;
}
memset(&g_FP_instance, 0, sizeof(g_FP_instance));
int output_cycle = 0;
char output_file[TFE_STRING_MAX] = {0};
char histogram_bins[256] = {0};
if (g_FP_instance.statsd_format != 1 && g_FP_instance.statsd_format != 2)
{
g_FP_instance.statsd_format = 1;
}
unsigned int nr_worker_threads = 0;
MESA_load_profile_uint_def(profile, "system", "nr_worker_threads", &nr_worker_threads, 1);
MESA_load_profile_int_def(profile, "STAT", "output_cycle", &output_cycle, 2);
MESA_load_profile_string_def(profile, "STAT", "output_file", output_file, sizeof(output_file), "log/future.fs4");
MESA_load_profile_string_def(profile, "STAT", "histogram_bins", histogram_bins, sizeof(histogram_bins), FP_HISTOGRAM_BINS);
MESA_htable_handle htable = MESA_htable_born();
value=0;
MESA_htable_set_opt(htable, MHO_SCREEN_PRINT_CTRL,&value,sizeof(value));
value=1;
MESA_htable_set_opt(htable, MHO_THREAD_SAFE, &value,sizeof(value));;
value=16;
MESA_htable_set_opt(htable, MHO_MUTEX_NUM, &value,sizeof(value));;
value=1024;
MESA_htable_set_opt(htable, MHO_HASH_SLOT_SIZE, &value,sizeof(value));;
int value = 0;
MESA_htable_set_opt(htable, MHO_SCREEN_PRINT_CTRL, &value, sizeof(value));
value = 1;
MESA_htable_set_opt(htable, MHO_THREAD_SAFE, &value, sizeof(value));
value = 16;
MESA_htable_set_opt(htable, MHO_MUTEX_NUM, &value, sizeof(value));
value = 1024;
MESA_htable_set_opt(htable, MHO_HASH_SLOT_SIZE, &value, sizeof(value));
MESA_htable_mature(htable);
g_FP_instance.name_table=htable;
g_FP_instance.name_table = htable;
screen_stat_handle_t fs=NULL;
const char* stat_path="log/future.fs2";
const char* app_name="FP";
fs=FS_create_handle();
FS_set_para(fs, APP_NAME, app_name, strlen(app_name)+1);
value=1;
FS_set_para(fs, OUTPUT_PROMETHEUS, &value, sizeof(value));
value=0;
FS_set_para(fs, FLUSH_BY_DATE, &value, sizeof(value));
FS_set_para(fs, OUTPUT_DEVICE, stat_path, strlen(stat_path)+1);
value=1;
FS_set_para(fs, PRINT_MODE, &value, sizeof(value));
value=1;
FS_set_para(fs, CREATE_THREAD, &value, sizeof(value));
FS_set_para(fs, STAT_CYCLE, &g_FP_instance.statsd_cycle, sizeof(g_FP_instance.statsd_cycle));
if(strlen(g_FP_instance.statsd_server_ip)>0 && g_FP_instance.statsd_server_port!=0)
{
FS_set_para(fs, STATS_SERVER_IP, g_FP_instance.statsd_server_ip, strlen(g_FP_instance.statsd_server_ip)+1);
FS_set_para(fs, STATS_SERVER_PORT, &(g_FP_instance.statsd_server_port), sizeof(g_FP_instance.statsd_server_port));
FS_set_para(fs, STATS_FORMAT, &(g_FP_instance.statsd_format), sizeof(g_FP_instance.statsd_format));
}
FS_set_para(fs, HISTOGRAM_GLOBAL_BINS, g_FP_instance.histogram_bins, strlen(g_FP_instance.histogram_bins)+1);
g_FP_instance.fsid_f_num=FS_register(fs, FS_STYLE_FIELD, g_FP_instance.favorite, "futures");
FS_start(fs);
g_FP_instance.fs_handle=fs;
g_is_FP_init=1;
g_FP_instance.fs_handle = fieldstat_easy_new(nr_worker_threads, "FP", NULL, 0);
fieldstat_easy_enable_auto_output(g_FP_instance.fs_handle, output_file, output_cycle);
g_FP_instance.fsid_f_num = fieldstat_easy_register_counter(g_FP_instance.fs_handle, "futures");
g_is_FP_init = 1;
return;
}
static struct promise * __future_to_promise(struct future * f)
@@ -147,7 +97,9 @@ static void __promise_destroy(struct promise *p)
{
p->cb_ctx_destroy(p->ctx);
}
if(!g_FP_instance.no_stats) FS_operate(g_FP_instance.fs_handle,g_FP_instance.fsid_f_num, 0, FS_OP_SUB, 1);
fieldstat_easy_counter_incrby(g_FP_instance.fs_handle, __currect_thread_id, g_FP_instance.fsid_f_num, NULL, 0, -1);
memset(p, 0, sizeof(struct promise));
free(p);
return;
@@ -172,58 +124,60 @@ void promise_allow_many_successes(struct promise *p)
struct field_get_set_args
{
MESA_htable_handle htable;
screen_stat_handle_t fs_handle;
struct fieldstat_easy *fs_handle;
int fsid_latency;
int fsid_failed;
};
static long field_get_set_cb(void * data, const uchar * key, uint size, void * user_arg)
static long field_get_set_cb(void *data, const uchar *key, uint size, void *user_arg)
{
struct field_get_set_args* args=(struct field_get_set_args*)user_arg;
int *field_id=NULL, ret=0;
const char* fail_str="_fail";
char buff[size+strlen(fail_str)+1];
if(data==NULL)
struct field_get_set_args *args = (struct field_get_set_args *)user_arg;
int *field_id = NULL, ret = 0;
const char *fail_str = "_fail";
char buff[size + strlen(fail_str) + 1];
if (data == NULL)
{
field_id=(int*)malloc(sizeof(int)*2);
snprintf(buff,sizeof(buff),"%s(ms)",(char*)key);
field_id[0]=FS_register_histogram(args->fs_handle, g_FP_instance.favorite, buff,
1, 30*1000,3);
args->fsid_latency=field_id[0];
snprintf(buff,sizeof(buff),"%s%s",(char*)key,fail_str);
field_id[1]=FS_register(args->fs_handle, FS_STYLE_FIELD, g_FP_instance.favorite, buff);
args->fsid_failed=field_id[1];
ret = MESA_htable_add(args->htable, key, size, (void*)field_id);
assert(ret>=0);
field_id = (int *)malloc(sizeof(int) * 2);
snprintf(buff, sizeof(buff), "%s(ms)", (char *)key);
field_id[0] = fieldstat_easy_register_histogram(args->fs_handle, buff, 1, 500000, 1);
args->fsid_latency = field_id[0];
snprintf(buff, sizeof(buff), "%s%s", (char *)key, fail_str);
field_id[1] = fieldstat_easy_register_counter(args->fs_handle, buff);
args->fsid_failed = field_id[1];
ret = MESA_htable_add(args->htable, key, size, (void *)field_id);
assert(ret >= 0);
}
else
{
field_id=(int*)data;
args->fsid_latency=field_id[0];
args->fsid_failed=field_id[1];
field_id = (int *)data;
args->fsid_latency = field_id[0];
args->fsid_failed = field_id[1];
}
(void)ret;
return 0;
}
struct future * future_create(const char* symbol, future_success_cb * cb_success, future_failed_cb * cb_failed, void * user)
struct future *future_create(const char *symbol, future_success_cb *cb_success, future_failed_cb *cb_failed, void *user)
{
struct promise * p = ALLOC(struct promise, 1);
struct promise *p = ALLOC(struct promise, 1);
p->f.user = user;
p->f.cb_success = cb_success;
p->f.cb_failed = cb_failed;
p->ref_cnt=1;
strncpy(p->f.symbol,symbol,sizeof(p->f.symbol));
if(!g_FP_instance.no_stats)
{
clock_gettime(CLOCK_MONOTONIC,&p->debug.create_time);
long cb_ret=0;
struct field_get_set_args args={.htable = g_FP_instance.name_table, .fs_handle = g_FP_instance.fs_handle};
MESA_htable_search_cb(g_FP_instance.name_table, (const unsigned char*)symbol, strlen(symbol), field_get_set_cb, &args, &cb_ret);
p->debug.fsid_latency=args.fsid_latency;
p->debug.fsid_failed=args.fsid_failed;
FS_operate(g_FP_instance.fs_handle,g_FP_instance.fsid_f_num, 0, FS_OP_ADD, 1);
}
p->ref_cnt = 1;
strncpy(p->f.symbol, symbol, sizeof(p->f.symbol));
clock_gettime(CLOCK_MONOTONIC, &p->debug.create_time);
long cb_ret = 0;
struct field_get_set_args args = {.htable = g_FP_instance.name_table, .fs_handle = g_FP_instance.fs_handle};
MESA_htable_search_cb(g_FP_instance.name_table, (const unsigned char *)symbol, strlen(symbol), field_get_set_cb, &args, &cb_ret);
p->debug.fsid_latency = args.fsid_latency;
p->debug.fsid_failed = args.fsid_failed;
fieldstat_easy_counter_incrby(g_FP_instance.fs_handle, __currect_thread_id, g_FP_instance.fsid_f_num, NULL, 0, 1);
return &p->f;
}
void future_set_timeout(struct future * f, struct timeval timeout)
@@ -255,29 +209,29 @@ void promise_finish(struct promise * p)
__promise_destroy(p);
}
}
static void fp_stat_latency(struct _future_promise_debug* debug, int is_success)
static void fp_stat_latency(struct _future_promise_debug *debug, int is_success)
{
struct timespec end;
long long jiffies_ms=0;
clock_gettime(CLOCK_MONOTONIC,&end);
if(is_success==1)
long long jiffies_ms = 0;
clock_gettime(CLOCK_MONOTONIC, &end);
if (is_success == 1)
{
debug->succ_times++;
}
else
{
FS_operate(g_FP_instance.fs_handle, debug->fsid_failed, 0, FS_OP_ADD, 1);
fieldstat_easy_counter_incrby(g_FP_instance.fs_handle, __currect_thread_id, debug->fsid_failed, NULL, 0, 1);
}
if(debug->succ_times<=1)
if (debug->succ_times <= 1)
{
jiffies_ms=(end.tv_sec-debug->create_time.tv_sec)*1000+(end.tv_nsec-debug->create_time.tv_nsec)/1000000;
FS_operate(g_FP_instance.fs_handle, debug->fsid_latency, 0, FS_OP_SET, jiffies_ms);
jiffies_ms = (end.tv_sec - debug->create_time.tv_sec) * 1000 + (end.tv_nsec - debug->create_time.tv_nsec) / 1000000;
fieldstat_easy_histogram_record(g_FP_instance.fs_handle, __currect_thread_id, debug->fsid_latency, NULL, 0, jiffies_ms);
}
return;
}
void promise_failed(struct promise * p, enum e_future_error error, const char * what)
{
if(!g_FP_instance.no_stats) fp_stat_latency(&p->debug, 0);
fp_stat_latency(&p->debug, 0);
if(!p->f.is_cancelled)
{
p->f.cb_failed(error, what, p->f.user);
@@ -291,7 +245,7 @@ void promise_failed(struct promise * p, enum e_future_error error, const char *
void promise_success(struct promise * p, void * result)
{
if(!g_FP_instance.no_stats) fp_stat_latency(&p->debug, 1);
fp_stat_latency(&p->debug, 1);
if(!p->f.is_cancelled)
{
p->f.cb_success(result, p->f.user);