diff --git a/common/include/kni_utils.h b/common/include/kni_utils.h index f7dc52f..bd2f447 100644 --- a/common/include/kni_utils.h +++ b/common/include/kni_utils.h @@ -97,6 +97,7 @@ enum kni_field{ KNI_FIELD_KEEPALIVE_REPLAY_ADD_SUCC, KNI_FIELD_KEEPALIVE_REPLAY_ADD_FAIL, KNI_FIELD_EXCEED_MTU, + KNI_FIELD_TFE_STATUS_BASE, }; struct kni_field_stat_handle{ diff --git a/conf/kni.conf b/conf/kni.conf index 56460d3..ad875a0 100644 --- a/conf/kni.conf +++ b/conf/kni.conf @@ -29,16 +29,19 @@ src_mac_addr = 00:0e:c6:d6:72:c1 [tfe0] mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = ens1f5 +ip_addr = keepalive_listen_port = 2476 [tfe1] mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = eth8 +ip_addr = keepalive_listen_port = 2477 [tfe2] mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = eth9 +ip_addr = keepalive_listen_port = 2478 [field_stat] @@ -79,5 +82,7 @@ mho_eliminate_type = LRU [tfe_mgr] keepalive_switch = 1 - +keepalive_idle = 2 +keepalive_intvl = 1 +keepalive_cnt = 3 keepalive_listen_eth = \ No newline at end of file diff --git a/entry/src/kni_entry.cpp b/entry/src/kni_entry.cpp index 08e8ede..79845ef 100644 --- a/entry/src/kni_entry.cpp +++ b/entry/src/kni_entry.cpp @@ -1632,6 +1632,11 @@ static struct kni_field_stat_handle * fs_init(const char *profile){ fs_handle->fields[KNI_FIELD_KEEPALIVE_REPLAY_ADD_FAIL] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "kaReplay_add_fail"); fs_handle->fields[KNI_FIELD_KEEPALIVE_REPLAY_ADD_SUCC] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "kaReplay_add_succ"); fs_handle->fields[KNI_FIELD_EXCEED_MTU] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "exceed_mtu"); + for(int i = 0; i < g_kni_handle->tfe_count; i++){ + char tfe_status[KNI_SYMBOL_MAX] = ""; + snprintf(tfe_status, sizeof(tfe_status), "tfe%d", i); + fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + i] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, tfe_status); + } fs_handle->handle = handle; FS_start(handle); return fs_handle; diff --git a/entry/src/tfe_mgr.cpp b/entry/src/tfe_mgr.cpp index 1926945..5c7f815 100644 --- a/entry/src/tfe_mgr.cpp +++ b/entry/src/tfe_mgr.cpp @@ -5,6 +5,8 @@ #include "kni_utils.h" #include "tfe_mgr.h" +extern struct kni_field_stat_handle *g_kni_fs_handle; + struct tfe_mgr{ pthread_rwlock_t rwlock; int tfe_alive_node_list[TFE_COUNT_MAX]; @@ -16,11 +18,10 @@ struct tfe_mgr{ struct thread_tfe_keepalive_args{ struct tfe_mgr* mgr; + int sockfd; int keepalive_idle; int keepalive_intvl; int keepalive_cnt; - uint32_t listen_ip; - int listen_port; int tfe_id; uint32_t tfe_ipaddr; }; @@ -59,7 +60,9 @@ static int tfe_mgr_alive_node_del(struct tfe_mgr *mgr, int tfe_id){ for(int j = i; j < mgr->tfe_alive_node_count - 1; j++){ mgr->tfe_alive_node_list[j] = mgr->tfe_alive_node_list[j + 1]; } + mgr->tfe_alive_node_count--; ret = 0; + FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + tfe_id], 0, FS_OP_SET, 0); goto out; out: @@ -83,6 +86,7 @@ static int tfe_mgr_alive_node_add(struct tfe_mgr *mgr, int tfe_id){ mgr->tfe_alive_node_list[mgr->tfe_alive_node_count] = tfe_id; mgr->tfe_alive_node_count++; ret = 0; + FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + tfe_id], 0, FS_OP_SET, 1); goto out; out: @@ -94,68 +98,26 @@ out: static void* thread_tfe_keepalive(void *args){ struct thread_tfe_keepalive_args *_args = (struct thread_tfe_keepalive_args*)args; struct tfe_mgr *mgr = _args->mgr; + int sockfd = _args->sockfd; int keepalive_idle = _args->keepalive_idle; int keepalive_intvl = _args->keepalive_intvl; int keepalive_cnt = _args->keepalive_cnt; - uint32_t listen_ip = _args->listen_ip; - int listen_port = _args->listen_port; int tfe_id = _args->tfe_id; uint32_t tfe_ipaddr = _args->tfe_ipaddr; void *logger = mgr->logger; FREE(&args); - - //create socket - struct sockaddr_in server_addr, client_addr; - socklen_t client_addr_len; - int ret; - int client_fd; - uint32_t client_ipaddr; - char client_ipaddr_str[INET_ADDRSTRLEN] = ""; - char *errmsg = NULL; - char buff[BUFF_SIZE_MAX]; - int sockfd = socket(AF_INET, SOCK_STREAM, 0); - if(sockfd < 0){ - KNI_LOG_ERROR(logger, "Failed at create tcp socket, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } - //set socketopt - ret = setsockopt(sockfd, SOL_TCP, TCP_KEEPIDLE, (void *)&keepalive_idle, sizeof(keepalive_idle)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPIDLE, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } - ret = setsockopt(sockfd, SOL_TCP, TCP_KEEPINTVL, (void *)&keepalive_intvl, sizeof(keepalive_intvl)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPINTVL, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } - ret = setsockopt(sockfd, SOL_TCP, TCP_KEEPCNT, (void *)&keepalive_cnt, sizeof(keepalive_cnt)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPCNT, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } - //bind - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; // IPv4 - server_addr.sin_addr.s_addr = listen_ip; - server_addr.sin_port = htons(listen_port); - ret = bind(sockfd, (const struct sockaddr *)&server_addr, sizeof(server_addr)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "Failed at bind tcp socket, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } - //listen - ret = listen(sockfd, 5); - if(ret < 0){ - KNI_LOG_ERROR(logger, "Failed at listen tcp socket, errno is %d, errmsg is %s", errno, strerror(errno)); - goto error_out; - } //accept + struct sockaddr_in client_addr; + socklen_t client_addr_len = sizeof(client_addr); + uint32_t client_ipaddr; + char client_ipaddr_str[INET_ADDRSTRLEN] = ""; + int flags, ret, client_fd; + char buff[BUFF_SIZE_MAX]; + char *errmsg = NULL; while(true){ - client_fd = accept(sockfd, (struct sockaddr *)&client_addr, &client_addr_len); if(client_fd < 0){ - KNI_LOG_ERROR(logger, "Failed at accept, errno is %d, errmsg is %s", errno, strerror(errno)); + KNI_LOG_ERROR(logger, "Failed at accept, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); continue; } client_ipaddr = client_addr.sin_addr.s_addr; @@ -165,19 +127,50 @@ static void* thread_tfe_keepalive(void *args){ close(client_fd); continue; } + //set socketopt keepalive + flags = 1; + ret = setsockopt(client_fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at set socketopt SO_KEEPALIVE, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + } + ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPIDLE, (void *)&keepalive_idle, sizeof(keepalive_idle)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPIDLE, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + } + ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPINTVL, (void *)&keepalive_intvl, sizeof(keepalive_intvl)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPINTVL, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + } + ret = setsockopt(client_fd, SOL_TCP, TCP_KEEPCNT, (void *)&keepalive_cnt, sizeof(keepalive_cnt)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at set socketopt TCP_KEEPCNT, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + } //succeed accpt: add alive node ret = tfe_mgr_alive_node_add(mgr, tfe_id); if(ret < 0){ errmsg = tfe_mgr_errmsg_get((enum tfe_mgr_errno)ret); - KNI_LOG_ERROR(logger, "Failed at add alive tfe node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); + KNI_LOG_ERROR(logger, "Failed at add tfe alive node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); + } + else{ + KNI_LOG_ERROR(logger, "Succeed at add tfe alive node, tfe_id is %d", tfe_id); } while(true){ ret = recv(client_fd, buff, sizeof(buff), 0); - if(ret < 0){ - //recv fin/rst - if(errno == ECONNREFUSED){ + if(ret == 0){ + KNI_LOG_ERROR(logger, "recv fin, del tfe alive node, tfe_id is %d", tfe_id); + break; + } + if(ret <= 0){ + if(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK){ + KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, continue recv, tfe_id is %d", errno, strerror(errno), tfe_id); + continue; + } + if(errno == ETIMEDOUT){ + KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, del tfe alive node, tfe_id is %d", errno, strerror(errno), tfe_id); break; } + KNI_LOG_ERROR(logger, "recv error, errno is %d, errmsg is %s, error_out, tfe_id is %d", errno, strerror(errno), tfe_id); + goto error_out; } } //recv fin: del alive node @@ -185,7 +178,10 @@ static void* thread_tfe_keepalive(void *args){ ret = tfe_mgr_alive_node_del(mgr, tfe_id); if(ret < 0){ errmsg = tfe_mgr_errmsg_get((enum tfe_mgr_errno)ret); - KNI_LOG_ERROR(logger, "Failed at del alive tfe node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); + KNI_LOG_ERROR(logger, "Failed at del tfe alive node, tfe_id is %d, errmsg is %s", tfe_id, errmsg); + } + else{ + KNI_LOG_ERROR(logger, "Succeed at del tfe alive node, tfe_id is %d", tfe_id); } } return NULL; @@ -194,6 +190,10 @@ error_out: if(sockfd > 0){ close(sockfd); } + if(client_fd > 0){ + close(client_fd); + } + KNI_LOG_ERROR(logger, "thread_tfe_keepalive exited, tfe_id is %d", tfe_id); return NULL; } @@ -203,6 +203,49 @@ void tfe_mgr_destroy(struct tfe_mgr* mgr){ FREE(&mgr); } } + +static int get_binded_sockfd(int tfe_id, uint32_t listen_ip, uint16_t listen_port, void *logger){ + //create socket + struct sockaddr_in server_addr; + int ret; + int sockfd = socket(AF_INET, SOCK_STREAM, 0); + int flag; + if(sockfd < 0){ + KNI_LOG_ERROR(logger, "Failed at create tcp socket, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + goto error_out; + } + flag = 1; + ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (void *)&flag, sizeof(flag)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at set socketopt SO_REUSEADDR, errno is %d, errmsg is %s, tfe_id is %d", errno, strerror(errno), tfe_id); + } + //bind + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; // IPv4 + server_addr.sin_addr.s_addr = listen_ip; + server_addr.sin_port = htons(listen_port); + ret = bind(sockfd, (const struct sockaddr *)&server_addr, sizeof(server_addr)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at bind tcp socket, port is %d, errno is %d, errmsg is %s, tfe_id is %d", + listen_port, errno, strerror(errno), tfe_id); + goto error_out; + } + //listen + ret = listen(sockfd, 5); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at listen tcp socket, errno is %d, errmsg is %s, tfe_id is %d, listen_port is %d", + errno, strerror(errno), tfe_id, listen_port); + goto error_out; + } + return sockfd; + +error_out: + if(sockfd > 0){ + close(sockfd); + } + return -1; +} + struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logger){ struct tfe_mgr* mgr = ALLOC(struct tfe_mgr, 1); mgr->logger = logger; @@ -263,14 +306,18 @@ struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logg ret, errno, strerror(errno), i, tfe_ipaddr_str); goto error_out; } + int sockfd = get_binded_sockfd(i, keepalive_listen_ip, keepalive_listen_port, logger); + if(sockfd < 0){ + KNI_LOG_ERROR(logger, "Failed at get binded sockfd, tfe_id is %d", i); + goto error_out; + } struct thread_tfe_keepalive_args *args = ALLOC(struct thread_tfe_keepalive_args, 1); args->mgr = mgr; args->keepalive_idle = keepalive_idle; + args->sockfd = sockfd; args->keepalive_intvl = keepalive_intvl; args->keepalive_cnt = keepalive_cnt; args->tfe_id = i; - args->listen_ip = keepalive_listen_ip; - args->listen_port = keepalive_listen_port; args->tfe_ipaddr = tfe_ipaddr; ret = pthread_create(&thread_id, NULL, thread_tfe_keepalive, (void *)args); if(unlikely(ret != 0)){