#include #include #include #include #include "kni_utils.h" #include "tfe_mgr.h" extern struct kni_field_stat_handle *g_kni_fs_handle; struct tfe_mgr{ pthread_rwlock_t rwlock; int tfe_alive_node_list[TFE_COUNT_MAX]; int tfe_alive_node_count; int tfe_node_count; int keepalive_switch; void *logger; }; struct thread_tfe_keepalive_args{ struct tfe_mgr* mgr; int sockfd; int keepalive_idle; int keepalive_intvl; int keepalive_cnt; int tfe_id; uint32_t tfe_ipaddr; }; enum tfe_mgr_errno{ TFE_MGR_NOT_EXISTED = -1, TFE_MGR_HAS_EXISTED = -2, TFE_MGR_EXCEED_MAX_COUNT = -3, }; static char* tfe_mgr_errmsg_get(enum tfe_mgr_errno _errno){ switch(_errno){ case TFE_MGR_NOT_EXISTED: return (char*)"tfe not existed"; case TFE_MGR_HAS_EXISTED: return (char*)"tfe has existed"; case TFE_MGR_EXCEED_MAX_COUNT: return (char*)"tfe exceed max count"; default: return (char*)"unknown"; } } static int tfe_mgr_alive_node_del(struct tfe_mgr *mgr, int tfe_id){ pthread_rwlock_wrlock(&(mgr->rwlock)); int i, ret; for(i = 0; i < mgr->tfe_alive_node_count; i++){ if(mgr->tfe_alive_node_list[i] == tfe_id){ break; } } if(i == mgr->tfe_alive_node_count){ ret = TFE_MGR_NOT_EXISTED; goto out; } for(int j = i; j < mgr->tfe_alive_node_count - 1; j++){ mgr->tfe_alive_node_list[j] = mgr->tfe_alive_node_list[j + 1]; } mgr->tfe_alive_node_count--; ret = 0; FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + tfe_id], 0, FS_OP_SET, 0); goto out; out: pthread_rwlock_unlock(&(mgr->rwlock)); return ret; } static int tfe_mgr_alive_node_add(struct tfe_mgr *mgr, int tfe_id){ pthread_rwlock_wrlock(&(mgr->rwlock)); int ret; for(int i = 0; i < mgr->tfe_alive_node_count; i++){ if(mgr->tfe_alive_node_list[i] == tfe_id){ ret = TFE_MGR_HAS_EXISTED; goto out; } } if(mgr->tfe_alive_node_count == TFE_COUNT_MAX){ ret = TFE_MGR_EXCEED_MAX_COUNT; goto out; } mgr->tfe_alive_node_list[mgr->tfe_alive_node_count] = tfe_id; mgr->tfe_alive_node_count++; ret = 0; FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + tfe_id], 0, FS_OP_SET, 1); goto out; out: pthread_rwlock_unlock(&(mgr->rwlock)); return ret; } static void* thread_tfe_keepalive(void *args){ struct thread_tfe_keepalive_args *_args = (struct thread_tfe_keepalive_args*)args; struct tfe_mgr *mgr = _args->mgr; int sockfd = _args->sockfd; int keepalive_idle = _args->keepalive_idle; int keepalive_intvl = _args->keepalive_intvl; int keepalive_cnt = _args->keepalive_cnt; int tfe_id = _args->tfe_id; uint32_t tfe_ipaddr = _args->tfe_ipaddr; void *logger = mgr->logger; FREE(&args); //accept struct sockaddr_in client_addr; socklen_t client_addr_len = sizeof(client_addr); uint32_t client_ipaddr; char client_ipaddr_str[INET_ADDRSTRLEN] = ""; int flags, ret, client_fd; char buff[BUFF_SIZE_MAX]; char *errmsg = NULL; while(true){ client_fd = accept(sockfd, (struct sockaddr *)&client_addr, &client_addr_len); if(client_fd < 0){ KNI_LOG_ERROR(logger, "Failed at accept, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); continue; } client_ipaddr = client_addr.sin_addr.s_addr; inet_ntop(AF_INET, &client_ipaddr, client_ipaddr_str, INET_ADDRSTRLEN); if(client_ipaddr != tfe_ipaddr){ KNI_LOG_ERROR(logger, "Receive connection not from tfe%d, client addr is %s", tfe_id, client_ipaddr); close(client_fd); continue; } //set socketopt keepalive flags = 1; ret = setsockopt(client_fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at set socketopt SO_KEEPALIVE, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); } ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPIDLE, (void *)&keepalive_idle, sizeof(keepalive_idle)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPIDLE, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); } ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPINTVL, (void *)&keepalive_intvl, sizeof(keepalive_intvl)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPINTVL, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); } ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPCNT, (void *)&keepalive_cnt, sizeof(keepalive_cnt)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPCNT, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); } //succeed accpt: add alive node ret = tfe_mgr_alive_node_add(mgr, tfe_id); if(ret < 0){ errmsg = tfe_mgr_errmsg_get((enum tfe_mgr_errno)ret); KNI_LOG_ERROR(logger, "Failed at add tfe alive node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); } else{ KNI_LOG_ERROR(logger, "Succeed at add tfe alive node, tfe_id is %d", tfe_id); } while(true){ ret = recv(client_fd, buff, sizeof(buff), 0); if(ret == 0){ KNI_LOG_ERROR(logger, "recv fin, del tfe alive node, tfe_id is %d", tfe_id); break; } if(ret <= 0){ if(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK){ KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, continue recv, tfe_id is %d", errno, strerror(errno), tfe_id); continue; } if(errno == ETIMEDOUT){ KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, del tfe alive node, tfe_id is %d", errno, strerror(errno), tfe_id); break; } KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, error_out, tfe_id is %d", errno, strerror(errno), tfe_id); goto error_out; } } //recv fin: del alive node close(client_fd); ret = tfe_mgr_alive_node_del(mgr, tfe_id); if(ret < 0){ errmsg = tfe_mgr_errmsg_get((enum tfe_mgr_errno)ret); KNI_LOG_ERROR(logger, "Failed at del tfe alive node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); } else{ KNI_LOG_ERROR(logger, "Succeed at del tfe alive node, tfe_id is %d", tfe_id); } } return NULL; error_out: if(sockfd > 0){ close(sockfd); } if(client_fd > 0){ close(client_fd); } KNI_LOG_ERROR(logger, "thread_tfe_keepalive exited, tfe_id is %d", tfe_id); return NULL; } void tfe_mgr_destroy(struct tfe_mgr* mgr){ if(mgr != NULL){ pthread_rwlock_destroy(&(mgr->rwlock)); FREE(&mgr); } } static int get_binded_sockfd(int tfe_id, uint32_t listen_ip, uint16_t listen_port, void *logger){ //create socket struct sockaddr_in server_addr; int ret; int sockfd = socket(AF_INET, SOCK_STREAM, 0); int flag; if(sockfd < 0){ KNI_LOG_ERROR(logger, "Failed at create tcp socket, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); goto error_out; } flag = 1; ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flag, sizeof(flag)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at set socketopt SO_REUSEADDR, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); } //bind memset(&server_addr, 0, sizeof(server_addr)); server_addr.sin_family = AF_INET; // IPv4 server_addr.sin_addr.s_addr = listen_ip; server_addr.sin_port = htons(listen_port); ret = bind(sockfd, (const struct sockaddr *)&server_addr, sizeof(server_addr)); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at bind tcp socket, port is %d, errno is %d, errmsg is %s, tfe_id is %d", listen_port, errno, strerror(errno), tfe_id); goto error_out; } //listen ret = listen(sockfd, 5); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at listen tcp socket, errno is %d, errmsg is %s, tfe_id is %d, listen_port is %d", errno, strerror(errno), tfe_id, listen_port); goto error_out; } return sockfd; error_out: if(sockfd > 0){ close(sockfd); } return -1; } struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logger){ struct tfe_mgr* mgr = ALLOC(struct tfe_mgr, 1); mgr->logger = logger; mgr->tfe_node_count = tfe_node_count; int ret; //load keepalive conf char section[KNI_SYMBOL_MAX] = "tfe_mgr"; MESA_load_profile_int_def(profile, section, "keepalive_switch", &(mgr->keepalive_switch), 0); KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n keepalive_switch: %d", section, mgr->keepalive_switch); if(mgr->keepalive_switch == 0){ return mgr; } int keepalive_idle, keepalive_cnt, keepalive_intvl; char keepalive_listen_eth[KNI_SYMBOL_MAX] = ""; uint32_t keepalive_listen_ip; int keepalive_listen_port; uint32_t tfe_ipaddr; char tfe_ipaddr_str[INET_ADDRSTRLEN]; pthread_t thread_id = -1; MESA_load_profile_int_def(profile, section, "keepalive_idle", &keepalive_idle, 2); MESA_load_profile_int_def(profile, section, "keepalive_intvl", &keepalive_intvl, 1); MESA_load_profile_int_def(profile, section, "keepalive_cnt", &keepalive_cnt, 3); ret = MESA_load_profile_string_nodef(profile, section, "keepalive_listen_eth", keepalive_listen_eth, sizeof(keepalive_listen_eth)); if(ret < 0){ KNI_LOG_ERROR(logger, "MESA_prof_load: keepalive_listen_eth not set, profile is %s, section is %s", profile, section); goto error_out; } KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n keepalive_idle: %d\n keepalive_intvl: %d\n keepalive_cnt: %d\n keepalive_listen_eth: %s", section, keepalive_idle, keepalive_intvl, keepalive_cnt, keepalive_listen_eth); ret = kni_ipv4_addr_get_by_eth(keepalive_listen_eth, &keepalive_listen_ip); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at get bind ipv4 addr, eth is %s", keepalive_listen_eth); goto error_out; } //rw_lock ret = pthread_rwlock_init(&(mgr->rwlock), NULL); if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at pthread_rwlock_init, errno is %d, errmsg is %s", errno, strerror(errno)); goto error_out; } //create tfe_keepalive thread for(int i = 0; i < tfe_node_count; i++){ snprintf(section, sizeof(section), "tfe%d", i); ret = MESA_load_profile_int_nodef(profile, section, "keepalive_listen_port", &keepalive_listen_port); if(ret < 0){ KNI_LOG_ERROR(logger, "MESA_prof_load: keepalive_listen_port not set, profile is %s, section is %s", profile, section); goto error_out; } ret = MESA_load_profile_string_nodef(profile, section, "ip_addr", tfe_ipaddr_str, sizeof(tfe_ipaddr_str)); if(ret < 0){ KNI_LOG_ERROR(logger, "MESA_prof_load: ip_addr not set, profile is %s, section is %s", profile, section); goto error_out; } KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n keepalive_listen_port: %d\n ip_addr: %s", section, keepalive_listen_port, tfe_ipaddr_str); ret = inet_pton(AF_INET, tfe_ipaddr_str, &tfe_ipaddr); if(ret != 1){ KNI_LOG_ERROR(logger, "Failed at inet_pton, ret is %d, errno is %d, errmsg is %s, tfe_id is %d, ip_addr is %s", ret, errno, strerror(errno), i, tfe_ipaddr_str); goto error_out; } int sockfd = get_binded_sockfd(i, keepalive_listen_ip, keepalive_listen_port, logger); if(sockfd < 0){ KNI_LOG_ERROR(logger, "Failed at get binded sockfd, tfe_id is %d", i); goto error_out; } struct thread_tfe_keepalive_args *args = ALLOC(struct thread_tfe_keepalive_args, 1); args->mgr = mgr; args->keepalive_idle = keepalive_idle; args->sockfd = sockfd; args->keepalive_intvl = keepalive_intvl; args->keepalive_cnt = keepalive_cnt; args->tfe_id = i; args->tfe_ipaddr = tfe_ipaddr; ret = pthread_create(&thread_id, NULL, thread_tfe_keepalive, (void *)args); if(unlikely(ret != 0)){ KNI_LOG_ERROR(logger, "Failed at pthread_create, thread_func is thread_tfe_keepalive, errno is %d, errmsg is %s", errno, strerror(errno)); FREE(&args); goto error_out; } } return mgr; error_out: tfe_mgr_destroy(mgr); return NULL; } int tfe_mgr_alive_node_get(struct tfe_mgr *mgr, int thread_seq){ if(mgr->keepalive_switch == 0){ return mgr->tfe_node_count > 0 ? thread_seq % mgr->tfe_node_count : -1; } pthread_rwlock_rdlock(&(mgr->rwlock)); int tfe_id = -1; if(mgr->tfe_alive_node_count > 0){ tfe_id = thread_seq % mgr->tfe_alive_node_count; } pthread_rwlock_unlock(&(mgr->rwlock)); return tfe_id; }