#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef TFE_CONFIG_KNI_SCM_SOCKET_FILE #define TFE_CONFIG_KNI_SCM_SOCKET_FILE "/var/run/.tfe_kmod_scm_socket" #endif /* The KNI and TFE communicate with each other by UNIX-based socket, * and the protocol between them is based on TLV format(Type-Length-Value). * The byte order for each entry in the protocol are Host-Ordered. * * I. Magic and Total counts of T-L-V tuples, at front of the SOCKET stream. * II. After Magic header, the stream consist of several T-L-V tuples. * * Note. Magic = 0x4d5a * Consider of the byte align problem, the minimum length of the value is 4bytes(32-bits). * * 0 1 2 3 * 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Magic | Total counts of TLV | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Type | Length | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Value | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Type | Length | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Value | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | ....... | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ struct kni_acceptor { /* INPUT */ struct tfe_proxy * proxy; const char * profile; void * logger; /* CONFIG */ char str_scm_socket_file[TFE_STRING_MAX]; /* PERSIST RUNTIME RESOURCE */ int fd_scm_socket; struct event_base * ev_base; struct event * ev_scm_socket; pthread_t thread; }; void __kni_event_cb(evutil_socket_t fd, short what, void * user) { struct kni_acceptor * __ctx = (struct kni_acceptor *) user; struct cmsghdr * __cmsghdr; struct tfe_proxy_accept_para __accept_para{}; int * __fds = NULL; assert(__ctx != NULL && __ctx->thread == pthread_self()); assert(what & EV_READ); /* We use IOVEC to recieve the fds make by KNI. * This is a kind of magic skill to share socket fds between two(or more) process. * http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html */ constexpr static int __TRANS_FDS_MAX = 2; constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int)); char __buffer[4096] = {0}; struct iovec __iovec[1]; struct msghdr __msghdr; char __cmptr[__CONTROLLEN]; __iovec[0].iov_base = __buffer; __iovec[0].iov_len = sizeof(__buffer); __msghdr.msg_iov = __iovec; __msghdr.msg_iovlen = 1; __msghdr.msg_name = NULL; __msghdr.msg_namelen = 0; __msghdr.msg_control = (void *) (__cmptr); __msghdr.msg_controllen = (size_t) __CONTROLLEN; ssize_t rd = recvmsg(fd, &__msghdr, 0); if (rd == -1 && (errno == EINTR || errno == EAGAIN)) { return; } else if (rd <= 0) { TFE_LOG_ERROR(__ctx->logger, "failed at recvmsg from scm socket: %s. ", strerror(errno)); goto __die; } __cmsghdr = CMSG_FIRSTHDR(&__msghdr); if (unlikely(__cmsghdr == NULL)) { TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_FIRSTHDR() from incoming fds."); goto __die; } __fds = (int *) (CMSG_DATA(__cmsghdr)); if (unlikely(__fds == NULL)) { TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_DATA() from incoming fds."); goto __die; } #if 0 if (unlikely(__kni_parse_tlv_data(__ctx, &__accept_para, __buffer, (size_t) rd) < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at parsing TLV format, close KNI connection."); goto __close_kni_connection; } #endif __accept_para.downstream_fd = __fds[0]; __accept_para.upstream_fd = __fds[1]; __accept_para.session_type = STREAM_PROTO_SSL; TFE_PROXY_STAT_INCREASE(STAT_FD_OPEN_BY_KNI_ACCEPT, 2); if (tfe_proxy_fds_accept(__ctx->proxy, &__accept_para) < 0) { goto __drop_recieved_fds; } return; __die: DIE("Broken kni scm socket connection, abort."); return; __drop_recieved_fds: TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 2); if (__fds != NULL) evutil_closesocket(__fds[0]); if (__fds != NULL) evutil_closesocket(__fds[1]); } void * kni_acceptor_event_thread_entry(void * args) { struct kni_acceptor * __ctx = (struct kni_acceptor *) args; assert(__ctx != NULL && __ctx->thread == pthread_self()); TFE_LOG_INFO(__ctx->logger, "kni acceptor thread is running."); event_base_dispatch(__ctx->ev_base); DIE("kni acceptor thread is exited, abort."); } void kni_acceptor_deinit(struct kni_acceptor * ctx) { if (ctx != NULL && ctx->ev_base != NULL) { event_base_free(ctx->ev_base); } if (ctx != NULL && ctx->fd_scm_socket != 0) { close(ctx->fd_scm_socket); } if (ctx != NULL) { free(ctx); } return; } struct kni_acceptor * kni_acceptor_init(struct tfe_proxy * proxy, const char * profile, void * logger) { struct kni_acceptor * __ctx = ALLOC(struct kni_acceptor, 1); struct sockaddr_un __sockaddr_un{}; int ret = 0; __ctx->proxy = proxy; __ctx->profile = profile; __ctx->logger = logger; /* Read the unix domain socket file, this file is used to recieve fds from KNI */ MESA_load_profile_string_def(profile, "kni", "scm_socket_file", __ctx->str_scm_socket_file, sizeof(__ctx->str_scm_socket_file), TFE_CONFIG_KNI_SCM_SOCKET_FILE); __sockaddr_un.sun_family = AF_UNIX; strncpy(__sockaddr_un.sun_path, __ctx->str_scm_socket_file, sizeof(__sockaddr_un.sun_path)); /* Create new event base, this event base will be dispatched at separated thread */ __ctx->ev_base = event_base_new(); if (unlikely(__ctx->ev_base == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. "); goto __errout; } __ctx->fd_scm_socket = socket(AF_UNIX, SOCK_DGRAM, 0); if (unlikely(__ctx->fd_scm_socket < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at create scm socket fd: %s", strerror(errno)); goto __errout; } ret = connect(__ctx->fd_scm_socket, (struct sockaddr *)&__sockaddr_un, sizeof(__sockaddr_un)); if (unlikely(ret < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at connecting to %s: %s", __sockaddr_un.sun_path, strerror(errno)); goto __errout; } __ctx->ev_scm_socket = event_new(__ctx->ev_base, __ctx->fd_scm_socket, EV_READ | EV_PERSIST, __kni_event_cb, __ctx); if (unlikely(__ctx->ev_scm_socket == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at setup READ event for scm socket fd %d.", __ctx->fd_scm_socket); goto __errout; } ret = event_add(__ctx->ev_scm_socket, NULL); if (unlikely(ret < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at adding scm socket event to evbase. "); goto __errout; } /* Create a thread to dispatch ctx->evbase */ ret = pthread_create(&__ctx->thread, NULL, kni_acceptor_event_thread_entry, (void *) __ctx); if (unlikely(ret < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event thread: %s", strerror(errno)); goto __errout; } TFE_LOG_INFO(__ctx->logger, "KNI scm socket file: %s", __ctx->str_scm_socket_file); return __ctx; __errout: kni_acceptor_deinit(__ctx); return NULL; }