diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 125e4da..fb2c8b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,13 +3,10 @@ variables: BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ INSTALL_PREFIX: "/opt/tsg/tfe" TESTING_VERSION_BUILD: 0 - BUILD_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master" BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux" stages: - build - - envelope - - upload .build_before_script: before_script: @@ -20,17 +17,6 @@ stages: - yum makecache - yum install -y elfutils-libelf-devel -.build_by_travis_for_centos7: - stage: build - image: $BUILD_IMAGE_CENTOS7 - extends: .build_before_script - script: - - yum install -y libmnl-devel - - yum install -y libnfnetlink-devel - - ./ci/travis.sh - tags: - - share - .build_by_travis_for_centos8: stage: build image: $BUILD_IMAGE_CENTOS8 @@ -46,94 +32,6 @@ stages: tags: - share -############################################################################### -# compile use image: build-env:master -############################################################################### - -branch_build_debug_for_centos7: - extends: .build_by_travis_for_centos7 - variables: - BUILD_TYPE: Debug - except: - - /^develop-.*$/i - - /^release-.*$/i - - tags - -branch_build_release_for_centos7: - variables: - BUILD_TYPE: RelWithDebInfo - extends: .build_by_travis_for_centos7 - except: - - /^develop-.*$/i - - /^release-.*$/i - - tags - -develop_build_debug_for_centos7: - extends: .build_by_travis_for_centos7 - variables: - TESTING_VERSION_BUILD: 1 - #UPLOAD_SYMBOL_FILES: 1 - BUILD_TYPE: Debug - # ASAN_OPTION: ADDRESS - PACKAGE: 1 - PULP3_REPO_NAME: tfe-testing-x86_64.el7 - PULP3_DIST_NAME: tfe-testing-x86_64.el7 - artifacts: - name: "tfe-develop-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - /^develop-.*$/i - - /^release-.*$/i - -develop_build_release_for_centos7: - extends: .build_by_travis_for_centos7 - variables: - TESTING_VERSION_BUILD: 1 - #UPLOAD_SYMBOL_FILES: 1 - # ASAN_OPTION: ADDRESS - BUILD_TYPE: RelWithDebInfo - PACKAGE: 1 - PULP3_REPO_NAME: tfe-testing-x86_64.el7 - PULP3_DIST_NAME: tfe-testing-x86_64.el7 - artifacts: - name: "tfe-develop-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - /^develop-.*$/i - - /^release-.*$/i - -release_build_debug_for_centos7: - variables: - #UPLOAD_SYMBOL_FILES: 1 - BUILD_TYPE: Debug - PACKAGE: 1 - PULP3_REPO_NAME: tfe-stable-x86_64.el7 - PULP3_DIST_NAME: tfe-stable-x86_64.el7 - extends: .build_by_travis_for_centos7 - artifacts: - name: "tfe-install-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - tags - -release_build_release_for_centos7: - variables: - BUILD_TYPE: RelWithDebInfo - #UPLOAD_SYMBOL_FILES: 1 - PACKAGE: 1 - PULP3_REPO_NAME: tfe-stable-x86_64.el7 - PULP3_DIST_NAME: tfe-stable-x86_64.el7 - extends: .build_by_travis_for_centos7 - artifacts: - name: "tfe-install-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - tags - ############################################################################### # compile use image: build-env:rockylinux ############################################################################### @@ -221,41 +119,3 @@ release_build_release_for_centos8: - build/*.rpm only: - tags - -############################################################################### -# envelope and upload -############################################################################### - -envelope_rpm: - stage: envelope - image: $BUILD_IMAGE_CENTOS7 - variables: - FEATURE_ID: 100 - APP_NAME_IN_RPM_SPEC: tfe - script: - - chmod +x ./ci/envelope_rpm.sh - - ./ci/envelope_rpm.sh - artifacts: - name: "tfe-pr-$CI_COMMIT_REF_NAME-release" - paths: - - build/*-pr-*.rpm - tags: - - envelope - only: - - tags - -upload_enveloped_rpm: - stage: upload - image: $BUILD_IMAGE_CENTOS7 - variables: - PULP3_REPO_NAME_EL7: tfe-stable-x86_64.el7 - PULP3_DIST_NAME_EL7: tfe-stable-x86_64.el7 - PULP3_REPO_NAME_EL8: tfe-stable-x86_64.el8 - PULP3_DIST_NAME_EL8: tfe-stable-x86_64.el8 - script: - - chmod +x ./ci/upload_enveloped_rpm.sh - - ./ci/upload_enveloped_rpm.sh - tags: - - share - only: - - tags \ No newline at end of file diff --git a/common/include/tfe_kafka_logger.h b/common/include/tfe_kafka_logger.h index 82db3d7..0ea0d20 100644 --- a/common/include/tfe_kafka_logger.h +++ b/common/include/tfe_kafka_logger.h @@ -28,12 +28,12 @@ typedef struct tfe_kafka_logger_s char topic_name[TOPIC_MAX][TFE_STRING_MAX]; char broker_list[TFE_STRING_MAX]; - rd_kafka_t *kafka_handle; + rd_kafka_t *kafka_handle[TOPIC_MAX]; rd_kafka_topic_t *kafka_topic[TOPIC_MAX]; } tfe_kafka_logger_t; -tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger); -int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_name, int topic_id, void *local_logger); +tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger); +int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger); void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger); int tfe_kafka_logger_send(tfe_kafka_logger_t *logger, int topic_id, const char *data, int len); diff --git a/common/src/tfe_kafka_logger.cpp b/common/src/tfe_kafka_logger.cpp index 26cfea6..28f4ce8 100644 --- a/common/src/tfe_kafka_logger.cpp +++ b/common/src/tfe_kafka_logger.cpp @@ -34,7 +34,7 @@ error: return INADDR_NONE; } -static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger) +static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_username, const char *sasl_passwd, const char *topic_name, void *local_logger) { int ret; char kafka_errstr[1024] = {0}; @@ -64,6 +64,13 @@ static rd_kafka_t *create_kafka_handle(const char *brokerlist, const char *sasl_ rd_kafka_conf_destroy(rconf); return NULL; } + ret = rd_kafka_conf_set(rconf, "client.id", topic_name, kafka_errstr, sizeof(kafka_errstr)); + if (ret != RD_KAFKA_CONF_OK) + { + TFE_LOG_ERROR(local_logger, "Error to set kafka \"client.id\", %s.", kafka_errstr); + rd_kafka_conf_destroy(rconf); + return NULL; + } if (strlen(sasl_username) > 0 && strlen(sasl_passwd) > 0) { @@ -109,11 +116,11 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam if(logger && logger->enable) { strncpy(logger->topic_name[topic_id], topic_name, sizeof(logger->topic_name[topic_id])-1); - logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle, topic_name, NULL); + logger->kafka_topic[topic_id] = rd_kafka_topic_new(logger->kafka_handle[topic_id], topic_name, NULL); if (logger->kafka_topic[topic_id] == NULL) { TFE_LOG_ERROR(local_logger, "Error to creat kafka topic: %s.", topic_name); - rd_kafka_destroy(logger->kafka_handle); + rd_kafka_destroy(logger->kafka_handle[topic_id]); free(logger); return -1; } @@ -121,22 +128,27 @@ int tfe_kafka_logger_topic_new(tfe_kafka_logger_t *logger, const char *topic_nam return 0; } -tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, const char *sasl_username, const char *sasl_passwd, void *local_logger) +tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, const char *brokerlist, void *local_logger) { char *override_sled_ip=NULL; + tfe_kafka_logger_t *logger = (tfe_kafka_logger_t *)calloc(1, sizeof(tfe_kafka_logger_t)); if (!logger) - return NULL; + { + return NULL; + } logger->enable = enable; if (!logger->enable) - return logger; + { + return logger; + } override_sled_ip = getenv("OVERRIDE_SLED_IP"); if(override_sled_ip != NULL) { strncpy(logger->local_ip_str, override_sled_ip, sizeof(logger->local_ip_str)-1); - goto create_kafka; + goto finish; } logger->local_ip_num = get_ip_by_eth_name(nic_name); @@ -147,32 +159,45 @@ tfe_kafka_logger_t *tfe_kafka_logger_create(int enable, const char *nic_name, co return NULL; } inet_ntop(AF_INET, &(logger->local_ip_num), logger->local_ip_str, sizeof(logger->local_ip_str)); +finish: + strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1); + return logger; +} -create_kafka: - strncpy(logger->broker_list, brokerlist, sizeof(logger->broker_list)-1); - logger->kafka_handle = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, local_logger); - if (logger->kafka_handle == NULL) +int tfe_logger_create_kafka_topic(tfe_kafka_logger_t *logger, const char *sasl_username, const char *sasl_passwd, const char *topic_name, int topic_id, void *local_logger) +{ + if(!logger->enable) + { + return 0; + } + + logger->kafka_handle[topic_id] = create_kafka_handle(logger->broker_list, sasl_username, sasl_passwd, topic_name, local_logger); + if (logger->kafka_handle[topic_id] == NULL) { TFE_LOG_ERROR(local_logger, "Error to creat kafka handler with brokerlist: %s.", logger->broker_list); free(logger); - return NULL; + return -1; } - return logger; + tfe_kafka_logger_topic_new(logger, topic_name, topic_id, logger); + return 0; } void tfe_kafka_logger_destroy(tfe_kafka_logger_t *logger) { if (logger) { - if (logger->kafka_handle) - rd_kafka_destroy(logger->kafka_handle); - - if (logger->kafka_topic[TOPIC_LOGGER]) - rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_LOGGER]); - - if (logger->kafka_topic[TOPIC_BUCKET]) - rd_kafka_topic_destroy(logger->kafka_topic[TOPIC_BUCKET]); + for(int i=0; ikafka_topic[i]) + { + rd_kafka_topic_destroy(logger->kafka_topic[i]); + } + if(logger->kafka_handle[i]) + { + rd_kafka_destroy(logger->kafka_handle[i]); + } + } free(logger); logger = NULL; } diff --git a/common/src/tfe_resource.cpp b/common/src/tfe_resource.cpp index d696ecf..c0c4e6d 100644 --- a/common/src/tfe_resource.cpp +++ b/common/src/tfe_resource.cpp @@ -175,7 +175,7 @@ error_out: static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char *section, void *logger) { - int enable = 0, vsystem_id = 0; + int ret=0, enable=0, vsystem_id=0; char nic_name[TFE_SYMBOL_MAX] = {0}; char brokerlist[TFE_STRING_MAX] = {0}; char logger_topic[TFE_STRING_MAX] = {0}; @@ -199,24 +199,26 @@ static tfe_kafka_logger_t *create_kafka_logger(const char *profile, const char * return NULL; } - kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, sasl_username, sasl_passwd, logger); - if (kafka_logger == NULL) - { - TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger."); - return NULL; - } + kafka_logger = tfe_kafka_logger_create(enable, nic_name, brokerlist, logger); + if (kafka_logger == NULL) + { + TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create kafka logger."); + return NULL; + } - int ret = tfe_kafka_logger_topic_new(kafka_logger, logger_topic, TOPIC_LOGGER, logger); - if(ret < 0) - { - return NULL; - } - - ret = tfe_kafka_logger_topic_new(kafka_logger, bucket_topic, TOPIC_BUCKET, logger); - if(ret < 0) - { - return NULL; - } + ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, logger_topic, TOPIC_LOGGER, logger); + if(ret < 0) + { + TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", logger_topic); + return NULL; + } + + ret = tfe_logger_create_kafka_topic(kafka_logger, sasl_username, sasl_passwd, bucket_topic, TOPIC_BUCKET, logger); + if(ret < 0) + { + TFE_LOG_ERROR(logger, "tfe kafka init failed, error to create %s topic.", bucket_topic); + return NULL; + } kafka_logger->t_vsys_id=vsystem_id; TFE_LOG_INFO(logger, "tfe kafka logger : %s", enable ? "ENABLE" : "DISABLE"); diff --git a/platform/src/ssl_fetch_cert.cpp b/platform/src/ssl_fetch_cert.cpp index fe7d22c..7a4a0de 100644 --- a/platform/src/ssl_fetch_cert.cpp +++ b/platform/src/ssl_fetch_cert.cpp @@ -45,10 +45,15 @@ struct ssl_mid_cert_ctx mid_cert_ctx; int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section) { + int ret=0; char topic_name[TFE_SYMBOL_MAX] = {0}; + char sasl_username[TFE_STRING_MAX] = {0}; + char sasl_passwd[TFE_STRING_MAX] = {0}; MESA_load_profile_int_def(profile, section, "mc_cache_enable", &mid_cert_ctx.enable, 0); - MESA_load_profile_string_def(profile, "tfe", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT"); + MESA_load_profile_string_def(profile, "kafka", "mc_cache_topic", topic_name, sizeof(topic_name), "PXY-EXCH-INTERMEDIA-CERT"); + MESA_load_profile_string_def(profile, "kafka", "SASL_USERNAME", sasl_username, sizeof(sasl_username), ""); + MESA_load_profile_string_def(profile, "kafka", "SASL_PASSWD", sasl_passwd, sizeof(sasl_passwd), ""); if(mid_cert_ctx.enable == 0) { @@ -60,7 +65,8 @@ int ssl_mid_cert_kafka_logger_create(const char *profile, const char *section) { return -1; } - int ret = tfe_kafka_logger_topic_new(mid_cert_ctx.g_kafka_logger, topic_name, TOPIC_MC_CACHE, g_default_logger); + + ret = tfe_logger_create_kafka_topic(mid_cert_ctx.g_kafka_logger, sasl_username, sasl_passwd, topic_name, TOPIC_MC_CACHE, g_default_logger); if(ret < 0) { return -1; diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index a1ab494..a2567c0 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -337,6 +337,17 @@ static int doh_get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *common_o return 0; } +int doh_add_host_to_object(cJSON *common_obj, const char *req_spec_host) +{ + unsigned int port; + char *format_host=ALLOC(char, strlen(req_spec_host)+1); + sscanf(req_spec_host, "%[^:]:%u", format_host, &port); + cJSON_AddStringToObject(common_obj, "doh_host", format_host); + cJSON_AddStringToObject(common_obj, "server_fqdn", format_host); + FREE(&format_host); + return 0; +} + int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, const struct tfe_stream *stream, struct doh_ctx *ctx) { struct doh_maat_rule_t *result = ctx->result; @@ -446,8 +457,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url); - cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host); - cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host); + doh_add_host_to_object(common_obj, http->req->req_spec.host); if(handle->effective_device_tag) { diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index e1cc528..51cefe9 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -175,6 +175,17 @@ static int get_ip_server_geolocation(struct tfe_cmsg * cmsg, cJSON *per_hit_obj) return 0; } +int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host) +{ + unsigned int port; + char *format_host=ALLOC(char, strlen(req_spec_host)+1); + sscanf(req_spec_host, "%[^:]:%u", format_host, &port); + cJSON_AddStringToObject(common_obj, "http_host", format_host); + cJSON_AddStringToObject(common_obj, "server_fqdn", format_host); + FREE(&format_host); + return 0; +} + int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) { const struct tfe_http_session* http=log_msg->http; @@ -300,8 +311,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url); - cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host); - cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host); + proxy_add_host_to_object(common_obj, http->req->req_spec.host); if(handle->effective_device_tag) {