gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-21787 Proxy初始化kafka topic时增加client.id参数

Browse Source 
TSG-21730 Proxy优化Manipulation日志中host和fqdn字段的格式
This commit is contained in:
fengweihao
2024-07-12 11:13:56 +08:00
parent 5eccfbb882
commit 88a7a8c5c4
7 changed files with 101 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
.gitlab-ci.yml
View File

@@ -3,13 +3,10 @@ variables:
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/opt/tsg/tfe" INSTALL_PREFIX: "/opt/tsg/tfe"
TESTING_VERSION_BUILD: 0 TESTING_VERSION_BUILD: 0
BUILD_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master"
BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux" BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
stages: stages:
- build - build
- envelope
- upload
.build_before_script: .build_before_script:
before_script: before_script:
@@ -20,17 +17,6 @@ stages:
- yum makecache - yum makecache
- yum install -y elfutils-libelf-devel - yum install -y elfutils-libelf-devel
.build_by_travis_for_centos7:
stage: build
image: $BUILD_IMAGE_CENTOS7
extends: .build_before_script
script:
- yum install -y libmnl-devel
- yum install -y libnfnetlink-devel
- ./ci/travis.sh
tags:
- share
.build_by_travis_for_centos8: .build_by_travis_for_centos8:
stage: build stage: build
image: $BUILD_IMAGE_CENTOS8 image: $BUILD_IMAGE_CENTOS8
@@ -46,94 +32,6 @@ stages:
tags: tags:
- share - share
###############################################################################
# compile use image: build-env:master
###############################################################################
branch_build_debug_for_centos7:
extends: .build_by_travis_for_centos7
variables:
BUILD_TYPE: Debug
except:
- /^develop-.*$/i
- /^release-.*$/i
- tags
branch_build_release_for_centos7:
variables:
BUILD_TYPE: RelWithDebInfo
extends: .build_by_travis_for_centos7
except:
- /^develop-.*$/i
- /^release-.*$/i
- tags
develop_build_debug_for_centos7:
extends: .build_by_travis_for_centos7
variables:
TESTING_VERSION_BUILD: 1
#UPLOAD_SYMBOL_FILES: 1
BUILD_TYPE: Debug
# ASAN_OPTION: ADDRESS
PACKAGE: 1
PULP3_REPO_NAME: tfe-testing-x86_64.el7
PULP3_DIST_NAME: tfe-testing-x86_64.el7
artifacts:
name: "tfe-develop-$CI_COMMIT_REF_NAME-debug"
paths:
- build/*.rpm
only:
- /^develop-.*$/i
- /^release-.*$/i
develop_build_release_for_centos7:
extends: .build_by_travis_for_centos7
variables:
TESTING_VERSION_BUILD: 1
#UPLOAD_SYMBOL_FILES: 1
# ASAN_OPTION: ADDRESS
BUILD_TYPE: RelWithDebInfo
PACKAGE: 1
PULP3_REPO_NAME: tfe-testing-x86_64.el7
PULP3_DIST_NAME: tfe-testing-x86_64.el7
artifacts:
name: "tfe-develop-$CI_COMMIT_REF_NAME-release"
paths:
- build/*.rpm
only:
- /^develop-.*$/i
- /^release-.*$/i
release_build_debug_for_centos7:
variables:
#UPLOAD_SYMBOL_FILES: 1
BUILD_TYPE: Debug
PACKAGE: 1
PULP3_REPO_NAME: tfe-stable-x86_64.el7
PULP3_DIST_NAME: tfe-stable-x86_64.el7
extends: .build_by_travis_for_centos7
artifacts:
name: "tfe-install-$CI_COMMIT_REF_NAME-debug"
paths:
- build/*.rpm
only:
- tags
release_build_release_for_centos7:
variables:
BUILD_TYPE: RelWithDebInfo
#UPLOAD_SYMBOL_FILES: 1
PACKAGE: 1
PULP3_REPO_NAME: tfe-stable-x86_64.el7
PULP3_DIST_NAME: tfe-stable-x86_64.el7
extends: .build_by_travis_for_centos7
artifacts:
name: "tfe-install-$CI_COMMIT_REF_NAME-release"
paths:
- build/*.rpm
only:
- tags
############################################################################### ###############################################################################
# compile use image: build-env:rockylinux # compile use image: build-env:rockylinux
############################################################################### ###############################################################################
@@ -221,41 +119,3 @@ release_build_release_for_centos8:
- build/*.rpm - build/*.rpm
only: only:
- tags - tags
###############################################################################
# envelope and upload
###############################################################################
envelope_rpm:
stage: envelope
image: $BUILD_IMAGE_CENTOS7
variables:
FEATURE_ID: 100
APP_NAME_IN_RPM_SPEC: tfe
script:
- chmod +x ./ci/envelope_rpm.sh
- ./ci/envelope_rpm.sh
artifacts:
name: "tfe-pr-$CI_COMMIT_REF_NAME-release"
paths:
- build/*-pr-*.rpm
tags:
- envelope
only:
- tags
upload_enveloped_rpm:
stage: upload
image: $BUILD_IMAGE_CENTOS7
variables:
PULP3_REPO_NAME_EL7: tfe-stable-x86_64.el7
PULP3_DIST_NAME_EL7: tfe-stable-x86_64.el7
PULP3_REPO_NAME_EL8: tfe-stable-x86_64.el8
PULP3_DIST_NAME_EL8: tfe-stable-x86_64.el8
script:
- chmod +x ./ci/upload_enveloped_rpm.sh
- ./ci/upload_enveloped_rpm.sh
tags:
- share
only:
- tags

6
common/include/tfe_kafka_logger.h
View File

@@ -28,12 +28,12 @@ typedef struct tfe_kafka_logger_s
char topic_name[TOPIC_MAX][TFE_STRING_MAX]; char topic_name[TOPIC_MAX][TFE_STRING_MAX];
char broker_list[TFE_STRING_MAX]; char broker_list[TFE_STRING_MAX];
rd_kafka_t *kafka_handle; rd_kafka_t *kafka_handle[TOPIC_MAX];
rd_kafka_topic_t *kafka_topic[TOPIC_MAX]; rd_kafka_topic_t *kafka_topic[TOPIC_MAX];
} tfe_kafka_logger_t; } tfe_kafka_logger_t;
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger); tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger);
int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_name, int topic_id, void *local_logger); int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger);
void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger); void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger);
int tfe_kafka_logger_send(tfe_kafka_logger_t *logger, int topic_id, const char *data, int len); int tfe_kafka_logger_send(tfe_kafka_logger_t *logger, int topic_id, const char *data, int len);

63
common/src/tfe_kafka_logger.cpp
View File

@@ -34,7 +34,7 @@ error:
return INADDR_NONE; return INADDR_NONE;
} }
static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger) static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, const char *topic_name, void *local_logger)
{ {
int ret; int ret;
char kafka_errstr[1024] = {0}; char kafka_errstr[1024] = {0};
@@ -64,6 +64,13 @@ static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_
rd_kafka_conf_destroy(rconf); rd_kafka_conf_destroy(rconf);
return NULL; return NULL;
} }
ret = rd_kafka_conf_set(rconf, "client.id", topic_name, kafka_errstr, sizeof(kafka_errstr));
if (ret != RD_KAFKA_CONF_OK)
{
TFE_LOG_ERROR(local_logger, "Error to set kafka \"client.id\", %s.", kafka_errstr);
rd_kafka_conf_destroy(rconf);
return NULL;
}
if (strlen(sasl_username) > 0 && strlen(sasl_passwd) > 0) if (strlen(sasl_username) > 0 && strlen(sasl_passwd) > 0)
{ {
@@ -109,11 +116,11 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam
if(logger && logger->enable) if(logger && logger->enable)
{ {
strncpy(logger->topic_name[topic_id], topic_name, sizeof(logger->topic_name[topic_id])-1); strncpy(logger->topic_name[topic_id], topic_name, sizeof(logger->topic_name[topic_id])-1);
logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle, topic_name, NULL); logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle[topic_id], topic_name, NULL);
if (logger->kafka_topic[topic_id] == NULL) if (logger->kafka_topic[topic_id] == NULL)
{ {
TFE_LOG_ERROR(local_logger, "Error to creat kafka topic: %s.", topic_name); TFE_LOG_ERROR(local_logger, "Error to creat kafka topic: %s.", topic_name);
rd_kafka_destroy(logger->kafka_handle); rd_kafka_destroy(logger->kafka_handle[topic_id]);
free(logger); free(logger);
return -1; return -1;
} }
@@ -121,22 +128,27 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam
return 0; return 0;
} }
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger) tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger)
{ {
char *override_sled_ip=NULL; char *override_sled_ip=NULL;
tfe_kafka_logger_t *logger = (tfe_kafka_logger_t *)calloc(1, sizeof(tfe_kafka_logger_t)); tfe_kafka_logger_t *logger = (tfe_kafka_logger_t *)calloc(1, sizeof(tfe_kafka_logger_t));
if (!logger) if (!logger)
{
return NULL; return NULL;
}
logger->enable = enable; logger->enable = enable;
if (!logger->enable) if (!logger->enable)
{
return logger; return logger;
}
override_sled_ip = getenv("OVERRIDE_SLED_IP"); override_sled_ip = getenv("OVERRIDE_SLED_IP");
if(override_sled_ip != NULL) if(override_sled_ip != NULL)
{ {
strncpy(logger->local_ip_str, override_sled_ip, sizeof(logger->local_ip_str)-1); strncpy(logger->local_ip_str, override_sled_ip, sizeof(logger->local_ip_str)-1);
goto create_kafka; goto finish;
} }
logger->local_ip_num = get_ip_by_eth_name(nic_name); logger->local_ip_num = get_ip_by_eth_name(nic_name);
@@ -147,32 +159,45 @@ tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, co
return NULL; return NULL;
} }
inet_ntop(AF_INET, &(logger->local_ip_num), logger->local_ip_str, sizeof(logger->local_ip_str)); inet_ntop(AF_INET, &(logger->local_ip_num), logger->local_ip_str, sizeof(logger->local_ip_str));
finish:
create_kafka:
strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1); strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1);
logger->kafka_handle = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, local_logger); return logger;
if (logger->kafka_handle == NULL) }
int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger)
{
if(!logger->enable)
{
return 0;
}
logger->kafka_handle[topic_id] = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, topic_name, local_logger);
if (logger->kafka_handle[topic_id] == NULL)
{ {
TFE_LOG_ERROR(local_logger, "Error to creat kafka handler with brokerlist: %s.", logger->broker_list); TFE_LOG_ERROR(local_logger, "Error to creat kafka handler with brokerlist: %s.", logger->broker_list);
free(logger); free(logger);
return NULL; return -1;
} }
return logger; tfe_kafka_logger_topic_new(logger, topic_name, topic_id, logger);
return 0;
} }
void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger) void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger)
{ {
if (logger) if (logger)
{ {
if (logger->kafka_handle) for(int i=0; i<TOPIC_MAX; i++)
rd_kafka_destroy(logger->kafka_handle); {
if(logger->kafka_topic[i])
if (logger->kafka_topic[TOPIC_LOGGER]) {
rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_LOGGER]); rd_kafka_topic_destroy(logger->kafka_topic[i]);
}
if (logger->kafka_topic[TOPIC_BUCKET])
rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_BUCKET]);
if(logger->kafka_handle[i])
{
rd_kafka_destroy(logger->kafka_handle[i]);
}
}
free(logger); free(logger);
logger = NULL; logger = NULL;
} }

10
common/src/tfe_resource.cpp
View File

@@ -175,7 +175,7 @@ error_out:
static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger) static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger)
{ {
int enable = 0, vsystem_id = 0; int ret=0, enable=0, vsystem_id=0;
char nic_name[TFE_SYMBOL_MAX] = {0}; char nic_name[TFE_SYMBOL_MAX] = {0};
char brokerlist[TFE_STRING_MAX] = {0}; char brokerlist[TFE_STRING_MAX] = {0};
char logger_topic[TFE_STRING_MAX] = {0}; char logger_topic[TFE_STRING_MAX] = {0};
@@ -199,22 +199,24 @@ static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *
return NULL; return NULL;
} }
kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, sasl_username, sasl_passwd, logger); kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, logger);
if (kafka_logger == NULL) if (kafka_logger == NULL)
{ {
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger."); TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger.");
return NULL; return NULL;
} }
int ret = tfe_kafka_logger_topic_new(kafka_logger, logger_topic, TOPIC_LOGGER, logger); ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, logger_topic, TOPIC_LOGGER, logger);
if(ret < 0) if(ret < 0)
{ {
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", logger_topic);
return NULL; return NULL;
} }
ret = tfe_kafka_logger_topic_new(kafka_logger, bucket_topic, TOPIC_BUCKET, logger); ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, bucket_topic, TOPIC_BUCKET, logger);
if(ret < 0) if(ret < 0)
{ {
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", bucket_topic);
return NULL; return NULL;
} }
kafka_logger->t_vsys_id=vsystem_id; kafka_logger->t_vsys_id=vsystem_id;

10
platform/src/ssl_fetch_cert.cpp
View File

@@ -45,10 +45,15 @@ struct ssl_mid_cert_ctx mid_cert_ctx;
int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section) int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section)
{ {
int ret=0;
char topic_name[TFE_SYMBOL_MAX] = {0}; char topic_name[TFE_SYMBOL_MAX] = {0};
char sasl_username[TFE_STRING_MAX] = {0};
char sasl_passwd[TFE_STRING_MAX] = {0};
MESA_load_profile_int_def(profile, section, "mc_cache_enable", &mid_cert_ctx.enable, 0); MESA_load_profile_int_def(profile, section, "mc_cache_enable", &mid_cert_ctx.enable, 0);
MESA_load_profile_string_def(profile, "tfe", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT"); MESA_load_profile_string_def(profile, "kafka", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT");
MESA_load_profile_string_def(profile, "kafka", "SASL_USERNAME", sasl_username, sizeof(sasl_username), "");
MESA_load_profile_string_def(profile, "kafka", "SASL_PASSWD", sasl_passwd, sizeof(sasl_passwd), "");
if(mid_cert_ctx.enable == 0) if(mid_cert_ctx.enable == 0)
{ {
@@ -60,7 +65,8 @@ int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section)
{ {
return -1; return -1;
} }
int ret = tfe_kafka_logger_topic_new(mid_cert_ctx.g_kafka_logger, topic_name, TOPIC_MC_CACHE, g_default_logger);
ret = tfe_logger_create_kafka_topic(mid_cert_ctx.g_kafka_logger, sasl_username, sasl_passwd, topic_name, TOPIC_MC_CACHE, g_default_logger);
if(ret < 0) if(ret < 0)
{ {
return -1; return -1;

14
plugin/business/doh/src/logger.cpp
View File

@@ -337,6 +337,17 @@ static int doh_get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o
return 0; return 0;
} }
int doh_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
sscanf(req_spec_host, "%[^:]:%u", format_host, &port);
cJSON_AddStringToObject(common_obj, "doh_host", format_host);
cJSON_AddStringToObject(common_obj, "server_fqdn", format_host);
FREE(&format_host);
return 0;
}
int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, const struct tfe_stream *stream, struct doh_ctx *ctx) int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, const struct tfe_stream *stream, struct doh_ctx *ctx)
{ {
struct doh_maat_rule_t *result = ctx->result; struct doh_maat_rule_t *result = ctx->result;
@@ -446,8 +457,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host); doh_add_host_to_object(common_obj, http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
if(handle->effective_device_tag) if(handle->effective_device_tag)
{ {

14
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -175,6 +175,17 @@ static int get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj)
return 0; return 0;
} }
int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
sscanf(req_spec_host, "%[^:]:%u", format_host, &port);
cJSON_AddStringToObject(common_obj, "http_host", format_host);
cJSON_AddStringToObject(common_obj, "server_fqdn", format_host);
FREE(&format_host);
return 0;
}
int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{ {
const struct tfe_http_session* http=log_msg->http; const struct tfe_http_session* http=log_msg->http;
@@ -300,8 +311,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host); proxy_add_host_to_object(common_obj, http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
if(handle->effective_device_tag) if(handle->effective_device_tag)
{ {