gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-21787 Proxy初始化kafka topic时增加client.id参数

Browse Source 
TSG-21730 Proxy优化Manipulation日志中host和fqdn字段的格式
This commit is contained in:
fengweihao
2024-07-12 11:13:56 +08:00
parent 5eccfbb882
commit 88a7a8c5c4
7 changed files with 101 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
.gitlab-ci.yml
View File

@@ -3,13 +3,10 @@ variables:
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/opt/tsg/tfe"
TESTING_VERSION_BUILD: 0
BUILD_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master"
BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
stages:
- build
- envelope
- upload
.build_before_script:
before_script:
@@ -20,17 +17,6 @@ stages:
- yum makecache
- yum install -y elfutils-libelf-devel
.build_by_travis_for_centos7:
stage: build
image: $BUILD_IMAGE_CENTOS7
extends: .build_before_script
script:
- yum install -y libmnl-devel
- yum install -y libnfnetlink-devel
- ./ci/travis.sh
tags:
- share
.build_by_travis_for_centos8:
stage: build
image: $BUILD_IMAGE_CENTOS8
@@ -46,94 +32,6 @@ stages:
tags:
- share
###############################################################################
# compile use image: build-env:master
###############################################################################
branch_build_debug_for_centos7:
extends: .build_by_travis_for_centos7
variables:
BUILD_TYPE: Debug
except:
- /^develop-.*$/i
- /^release-.*$/i
- tags
branch_build_release_for_centos7:
variables:
BUILD_TYPE: RelWithDebInfo
extends: .build_by_travis_for_centos7
except:
- /^develop-.*$/i
- /^release-.*$/i
- tags
develop_build_debug_for_centos7:
extends: .build_by_travis_for_centos7
variables:
TESTING_VERSION_BUILD: 1
#UPLOAD_SYMBOL_FILES: 1
BUILD_TYPE: Debug
# ASAN_OPTION: ADDRESS
PACKAGE: 1
PULP3_REPO_NAME: tfe-testing-x86_64.el7
PULP3_DIST_NAME: tfe-testing-x86_64.el7
artifacts:
name: "tfe-develop-$CI_COMMIT_REF_NAME-debug"
paths:
- build/*.rpm
only:
- /^develop-.*$/i
- /^release-.*$/i
develop_build_release_for_centos7:
extends: .build_by_travis_for_centos7
variables:
TESTING_VERSION_BUILD: 1
#UPLOAD_SYMBOL_FILES: 1
# ASAN_OPTION: ADDRESS
BUILD_TYPE: RelWithDebInfo
PACKAGE: 1
PULP3_REPO_NAME: tfe-testing-x86_64.el7
PULP3_DIST_NAME: tfe-testing-x86_64.el7
artifacts:
name: "tfe-develop-$CI_COMMIT_REF_NAME-release"
paths:
- build/*.rpm
only:
- /^develop-.*$/i
- /^release-.*$/i
release_build_debug_for_centos7:
variables:
#UPLOAD_SYMBOL_FILES: 1
BUILD_TYPE: Debug
PACKAGE: 1
PULP3_REPO_NAME: tfe-stable-x86_64.el7
PULP3_DIST_NAME: tfe-stable-x86_64.el7
extends: .build_by_travis_for_centos7
artifacts:
name: "tfe-install-$CI_COMMIT_REF_NAME-debug"
paths:
- build/*.rpm
only:
- tags
release_build_release_for_centos7:
variables:
BUILD_TYPE: RelWithDebInfo
#UPLOAD_SYMBOL_FILES: 1
PACKAGE: 1
PULP3_REPO_NAME: tfe-stable-x86_64.el7
PULP3_DIST_NAME: tfe-stable-x86_64.el7
extends: .build_by_travis_for_centos7
artifacts:
name: "tfe-install-$CI_COMMIT_REF_NAME-release"
paths:
- build/*.rpm
only:
- tags
###############################################################################
# compile use image: build-env:rockylinux
###############################################################################
@@ -221,41 +119,3 @@ release_build_release_for_centos8:
- build/*.rpm
only:
- tags
###############################################################################
# envelope and upload
###############################################################################
envelope_rpm:
stage: envelope
image: $BUILD_IMAGE_CENTOS7
variables:
FEATURE_ID: 100
APP_NAME_IN_RPM_SPEC: tfe
script:
- chmod +x ./ci/envelope_rpm.sh
- ./ci/envelope_rpm.sh
artifacts:
name: "tfe-pr-$CI_COMMIT_REF_NAME-release"
paths:
- build/*-pr-*.rpm
tags:
- envelope
only:
- tags
upload_enveloped_rpm:
stage: upload
image: $BUILD_IMAGE_CENTOS7
variables:
PULP3_REPO_NAME_EL7: tfe-stable-x86_64.el7
PULP3_DIST_NAME_EL7: tfe-stable-x86_64.el7
PULP3_REPO_NAME_EL8: tfe-stable-x86_64.el8
PULP3_DIST_NAME_EL8: tfe-stable-x86_64.el8
script:
- chmod +x ./ci/upload_enveloped_rpm.sh
- ./ci/upload_enveloped_rpm.sh
tags:
- share
only:
- tags

6
common/include/tfe_kafka_logger.h
View File

@@ -28,12 +28,12 @@ typedef struct tfe_kafka_logger_s
char topic_name[TOPIC_MAX][TFE_STRING_MAX];
char broker_list[TFE_STRING_MAX];
rd_kafka_t *kafka_handle;
rd_kafka_t *kafka_handle[TOPIC_MAX];
rd_kafka_topic_t *kafka_topic[TOPIC_MAX];
} tfe_kafka_logger_t;
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger);
int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_name, int topic_id, void *local_logger);
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger);
int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger);
void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger);
int tfe_kafka_logger_send(tfe_kafka_logger_t *logger, int topic_id, const char *data, int len);

67
common/src/tfe_kafka_logger.cpp
View File

@@ -34,7 +34,7 @@ error:
return INADDR_NONE;
}
static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger)
static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, const char *topic_name, void *local_logger)
{
int ret;
char kafka_errstr[1024] = {0};
@@ -64,6 +64,13 @@ static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_
rd_kafka_conf_destroy(rconf);
return NULL;
}
ret = rd_kafka_conf_set(rconf, "client.id", topic_name, kafka_errstr, sizeof(kafka_errstr));
if (ret != RD_KAFKA_CONF_OK)
{
TFE_LOG_ERROR(local_logger, "Error to set kafka \"client.id\", %s.", kafka_errstr);
rd_kafka_conf_destroy(rconf);
return NULL;
}
if (strlen(sasl_username) > 0 && strlen(sasl_passwd) > 0)
{
@@ -109,11 +116,11 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam
if(logger && logger->enable)
{
strncpy(logger->topic_name[topic_id], topic_name, sizeof(logger->topic_name[topic_id])-1);
logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle, topic_name, NULL);
logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle[topic_id], topic_name, NULL);
if (logger->kafka_topic[topic_id] == NULL)
{
TFE_LOG_ERROR(local_logger, "Error to creat kafka topic: %s.", topic_name);
rd_kafka_destroy(logger->kafka_handle);
rd_kafka_destroy(logger->kafka_handle[topic_id]);
free(logger);
return -1;
}
@@ -121,22 +128,27 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam
return 0;
}
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger)
tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger)
{
char *override_sled_ip=NULL;
tfe_kafka_logger_t *logger = (tfe_kafka_logger_t *)calloc(1, sizeof(tfe_kafka_logger_t));
if (!logger)
return NULL;
{
return NULL;
}
logger->enable = enable;
if (!logger->enable)
return logger;
{
return logger;
}
override_sled_ip = getenv("OVERRIDE_SLED_IP");
if(override_sled_ip != NULL)
{
strncpy(logger->local_ip_str, override_sled_ip, sizeof(logger->local_ip_str)-1);
goto create_kafka;
goto finish;
}
logger->local_ip_num = get_ip_by_eth_name(nic_name);
@@ -147,32 +159,45 @@ tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, co
return NULL;
}
inet_ntop(AF_INET, &(logger->local_ip_num), logger->local_ip_str, sizeof(logger->local_ip_str));
finish:
strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1);
return logger;
}
create_kafka:
strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1);
logger->kafka_handle = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, local_logger);
if (logger->kafka_handle == NULL)
int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger)
{
if(!logger->enable)
{
return 0;
}
logger->kafka_handle[topic_id] = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, topic_name, local_logger);
if (logger->kafka_handle[topic_id] == NULL)
{
TFE_LOG_ERROR(local_logger, "Error to creat kafka handler with brokerlist: %s.", logger->broker_list);
free(logger);
return NULL;
return -1;
}
return logger;
tfe_kafka_logger_topic_new(logger, topic_name, topic_id, logger);
return 0;
}
void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger)
{
if (logger)
{
if (logger->kafka_handle)
rd_kafka_destroy(logger->kafka_handle);
if (logger->kafka_topic[TOPIC_LOGGER])
rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_LOGGER]);
if (logger->kafka_topic[TOPIC_BUCKET])
rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_BUCKET]);
for(int i=0; i<TOPIC_MAX; i++)
{
if(logger->kafka_topic[i])
{
rd_kafka_topic_destroy(logger->kafka_topic[i]);
}
if(logger->kafka_handle[i])
{
rd_kafka_destroy(logger->kafka_handle[i]);
}
}
free(logger);
logger = NULL;
}

38
common/src/tfe_resource.cpp
View File

@@ -175,7 +175,7 @@ error_out:
static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger)
{
int enable = 0, vsystem_id = 0;
int ret=0, enable=0, vsystem_id=0;
char nic_name[TFE_SYMBOL_MAX] = {0};
char brokerlist[TFE_STRING_MAX] = {0};
char logger_topic[TFE_STRING_MAX] = {0};
@@ -199,24 +199,26 @@ static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *
return NULL;
}
kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, sasl_username, sasl_passwd, logger);
if (kafka_logger == NULL)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger.");
return NULL;
}
kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, logger);
if (kafka_logger == NULL)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger.");
return NULL;
}
int ret = tfe_kafka_logger_topic_new(kafka_logger, logger_topic, TOPIC_LOGGER, logger);
if(ret < 0)
{
return NULL;
}
ret = tfe_kafka_logger_topic_new(kafka_logger, bucket_topic, TOPIC_BUCKET, logger);
if(ret < 0)
{
return NULL;
}
ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, logger_topic, TOPIC_LOGGER, logger);
if(ret < 0)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", logger_topic);
return NULL;
}
ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, bucket_topic, TOPIC_BUCKET, logger);
if(ret < 0)
{
TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", bucket_topic);
return NULL;
}
kafka_logger->t_vsys_id=vsystem_id;
TFE_LOG_INFO(logger, "tfe kafka logger : %s", enable ? "ENABLE" : "DISABLE");

10
platform/src/ssl_fetch_cert.cpp
View File

@@ -45,10 +45,15 @@ struct ssl_mid_cert_ctx mid_cert_ctx;
int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section)
{
int ret=0;
char topic_name[TFE_SYMBOL_MAX] = {0};
char sasl_username[TFE_STRING_MAX] = {0};
char sasl_passwd[TFE_STRING_MAX] = {0};
MESA_load_profile_int_def(profile, section, "mc_cache_enable", &mid_cert_ctx.enable, 0);
MESA_load_profile_string_def(profile, "tfe", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT");
MESA_load_profile_string_def(profile, "kafka", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT");
MESA_load_profile_string_def(profile, "kafka", "SASL_USERNAME", sasl_username, sizeof(sasl_username), "");
MESA_load_profile_string_def(profile, "kafka", "SASL_PASSWD", sasl_passwd, sizeof(sasl_passwd), "");
if(mid_cert_ctx.enable == 0)
{
@@ -60,7 +65,8 @@ int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section)
{
return -1;
}
int ret = tfe_kafka_logger_topic_new(mid_cert_ctx.g_kafka_logger, topic_name, TOPIC_MC_CACHE, g_default_logger);
ret = tfe_logger_create_kafka_topic(mid_cert_ctx.g_kafka_logger, sasl_username, sasl_passwd, topic_name, TOPIC_MC_CACHE, g_default_logger);
if(ret < 0)
{
return -1;

14
plugin/business/doh/src/logger.cpp
View File

@@ -337,6 +337,17 @@ static int doh_get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o
return 0;
}
int doh_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
sscanf(req_spec_host, "%[^:]:%u", format_host, &port);
cJSON_AddStringToObject(common_obj, "doh_host", format_host);
cJSON_AddStringToObject(common_obj, "server_fqdn", format_host);
FREE(&format_host);
return 0;
}
int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, const struct tfe_stream *stream, struct doh_ctx *ctx)
{
struct doh_maat_rule_t *result = ctx->result;
@@ -446,8 +457,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
doh_add_host_to_object(common_obj, http->req->req_spec.host);
if(handle->effective_device_tag)
{

14
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -175,6 +175,17 @@ static int get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj)
return 0;
}
int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
sscanf(req_spec_host, "%[^:]:%u", format_host, &port);
cJSON_AddStringToObject(common_obj, "http_host", format_host);
cJSON_AddStringToObject(common_obj, "server_fqdn", format_host);
FREE(&format_host);
return 0;
}
int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{
const struct tfe_http_session* http=log_msg->http;
@@ -300,8 +311,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
proxy_add_host_to_object(common_obj, http->req->req_spec.host);
if(handle->effective_device_tag)
{