#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT #define TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT "/var/run/.tfe_kni_acceptor_handler" #endif /* The KNI and TFE communicate with each other by UNIX-based socket, * and the protocol between them is based on TLV format(Type-Length-Value). * The byte order for each entry in the protocol are Host-Ordered. * * I. Magic and Total counts of T-L-V tuples, at front of the SOCKET stream. * II. After Magic header, the stream consist of several T-L-V tuples. * * Note. Magic = 0x4d5a * Consider of the byte align problem, the minimum length of the value is 4bytes(32-bits). * * 0 1 2 3 * 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Magic | Total counts of TLV | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Type | Length | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Value | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Type | Length | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | Value | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * | ....... | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ enum { KNI_TLV_MAGIC = 0x4d5a }; enum KNI_TLV_TYPE { KNI_TLV_TYPE_PROTOCOL = 0x0001, KNI_TLV_TYPE_KEYRING_ID = 0x0002 }; enum KNI_TLV_VALUE { KNI_TLV_VALUE_HTTP = 0x1, KNI_TLV_VALUE_SSL = 0x2, }; struct kni_tlv_header { uint16_t magic; uint16_t counts; }; struct kni_tlv_info { uint16_t type; uint16_t len; union { uint8_t value_as_raw[0]; uint16_t value_as_uint16; uint32_t value_as_uint32; uint16_t value_as_uint64; }; }; struct kni_acceptor { /* INPUT */ struct tfe_proxy * proxy; const char * profile; void * logger; /* CONFIG */ char str_unixdomain_file[TFE_STRING_MAX]; /* PERSIST RUNTIME RESOURCE */ int fd_unixdomain; struct event_base * ev_base; struct evconnlistener * ev_listener; pthread_t thread; /* PEER CONNECTION RESOUCE * should close by __kni_conn_close() */ struct event * ev_kni_conn; int fd_kni_conn; pid_t pid_kni_conn; }; void __kni_conn_close(struct kni_acceptor * ctx) { if (ctx->fd_kni_conn != 0) { close(ctx->fd_kni_conn); ctx->fd_kni_conn = 0; } if (ctx->ev_kni_conn != NULL) { event_free(ctx->ev_kni_conn); ctx->ev_kni_conn = NULL; } if (ctx->pid_kni_conn != 0) { ctx->pid_kni_conn = 0; } } static int __kni_parse_tlv_data(struct kni_acceptor * ctx, struct tfe_proxy_accept_para * out_para, const char * data, size_t sz_data) { const char * __cursor = data; size_t __left_data_len = sz_data; /* Parse the STREAM header */ struct kni_tlv_header * tlv_header = (struct kni_tlv_header *) __cursor; if (__left_data_len < sizeof(struct kni_tlv_header)) { TFE_LOG_ERROR(ctx->logger, "Invalid KNI TLV header format, length is not enough."); return -1; } if (tlv_header->magic != KNI_TLV_MAGIC) { TFE_LOG_ERROR(ctx->logger, "Invalid KNI TLV header magic number %x", tlv_header->magic); return -2; } unsigned int nr_tlv_tuples = tlv_header->counts; assert(nr_tlv_tuples != 0 && nr_tlv_tuples < 255); __left_data_len -= sizeof(struct kni_tlv_header); __cursor += sizeof(struct kni_tlv_header); for (unsigned int iter_tlv_tuples = 0; iter_tlv_tuples < nr_tlv_tuples; iter_tlv_tuples++) { struct kni_tlv_info * tlv_info = (struct kni_tlv_info *) __cursor; size_t sz_tlv_info = tlv_info->len + sizeof(struct kni_tlv_info); if (__left_data_len < sz_tlv_info) { TFE_LOG_ERROR(ctx->logger, "TLV info declares invalid value length, Too long."); return -3; } switch (tlv_info->type) { /* VALUE is uint32_t, length is 4 */ case KNI_TLV_TYPE_PROTOCOL: { uint32_t __value = tlv_info->value_as_uint32; if (__value == KNI_TLV_VALUE_HTTP) { out_para->session_type = STREAM_PROTO_PLAIN; } if (__value == KNI_TLV_VALUE_SSL) { out_para->session_type = STREAM_PROTO_SSL; } assert(tlv_info->len == sizeof(uint32_t)); break; } /* VALUE is uint32_t, length is 4 */ case KNI_TLV_TYPE_KEYRING_ID: { uint32_t __value = tlv_info->value_as_uint32; out_para->keyring_id = __value; assert(tlv_info->len == sizeof(uint32_t)); break; } default: assert(0); } __cursor += sz_tlv_info; __left_data_len -= sz_tlv_info; } return 0; } void __kni_event_cb(evutil_socket_t fd, short what, void * user) { struct kni_acceptor * __ctx = (struct kni_acceptor *) user; struct cmsghdr * __cmsghdr; struct tfe_proxy_accept_para __accept_para{}; int * __fds = NULL; assert(__ctx != NULL && __ctx->thread == pthread_self()); assert(what & EV_READ); /* We use IOVEC to recieve the fds make by KNI. * This is a kind of magic skill to share socket fds between two(or more) process. * http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html */ constexpr static int __TRANS_FDS_MAX = 2; constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int)); char __buffer[512] = {0}; struct iovec __iovec[1]; struct msghdr __msghdr; char __cmptr[__CONTROLLEN]; __iovec[0].iov_base = __buffer; __iovec[0].iov_len = sizeof(__buffer); __msghdr.msg_iov = __iovec; __msghdr.msg_iovlen = 1; __msghdr.msg_name = NULL; __msghdr.msg_namelen = 0; __msghdr.msg_control = (void *) (__cmptr); __msghdr.msg_controllen = (size_t) __CONTROLLEN; ssize_t rd = recvmsg(fd, &__msghdr, 0); if (rd == -1 && (errno == EINTR || errno == EAGAIN)) { return; } else if (rd < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at recving fds from KNI connection: %s. ", strerror(errno)); goto __close_kni_connection; } else if (rd == 0) { TFE_LOG_INFO(__ctx->logger, "KNI disconnect (PID = %u). ", __ctx->pid_kni_conn); goto __close_kni_connection; } __cmsghdr = CMSG_FIRSTHDR(&__msghdr); __fds = (int *) (CMSG_DATA(__cmsghdr)); if (unlikely(__fds == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at fetch CMSG_DATA() from incoming fds, close KNI connection."); goto __close_kni_connection; } if (unlikely(__kni_parse_tlv_data(__ctx, &__accept_para, __buffer, (size_t) rd) < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at parsing TLV format, close KNI connection."); goto __close_kni_connection; } __accept_para.downstream_fd = __fds[0]; __accept_para.upstream_fd = __fds[1]; TFE_PROXY_STAT_INCREASE(STAT_FD_OPEN_BY_KNI_ACCEPT, 2); if (tfe_proxy_fds_accept(__ctx->proxy, &__accept_para) < 0) { goto __drop_recieved_fds; } return; __close_kni_connection: __kni_conn_close(__ctx); __drop_recieved_fds: TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 2); if (__fds != NULL) evutil_closesocket(__fds[0]); if (__fds != NULL) evutil_closesocket(__fds[1]); } void __kni_listener_accept_cb(struct evconnlistener * listener, evutil_socket_t fd, struct sockaddr * sk_addr, int sk_len, void * user) { struct kni_acceptor * __ctx = (struct kni_acceptor *) user; struct event * __event = NULL; struct ucred __cr{}; socklen_t __cr_len = sizeof(struct ucred); int ret = 0; /* There is only one KNI process can connect to TFE. * If ev_kni_conn is not NULL, there's already a KNI connected to TFE. * We need to refuse this connection */ if (unlikely(__ctx->ev_kni_conn != NULL)) { TFE_LOG_ERROR(__ctx->logger, "One KNI(PID = %d) has been connected to our program, " "close this connection", __ctx->pid_kni_conn); evutil_closesocket(fd); return; } /* Get Peer's PID */ if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, (void *) &__cr, &__cr_len) < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at getsockopt(SO_PEERCRED) for fd %d, close this connection", fd); goto __close_this_connection; } __event = event_new(__ctx->ev_base, fd, EV_READ | EV_PERSIST, __kni_event_cb, __ctx); if (unlikely(__event == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event for fd %d, close this connection.", fd); goto __close_this_connection; } ret = event_add(__event, NULL); if (unlikely(ret < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at adding event to evbase, close this connection. "); goto __close_this_connection; } __ctx->fd_kni_conn = fd; __ctx->ev_kni_conn = __event; __ctx->pid_kni_conn = __cr.pid; TFE_LOG_INFO(__ctx->logger, "KNI connected (PID = %u, fd = %d). ", __ctx->pid_kni_conn, __ctx->fd_kni_conn); return; __close_this_connection: __kni_conn_close(__ctx); } void * __kni_listener_thread_entry(void * args) { struct kni_acceptor * __ctx = (struct kni_acceptor *) args; assert(__ctx != NULL && __ctx->thread == pthread_self()); TFE_LOG_DEBUG(__ctx->logger, "Starting KNI listener thread..."); event_base_dispatch(__ctx->ev_base); TFE_LOG_DEBUG(__ctx->logger, "Stoping KNI listener thread..."); return (void *) NULL; } void kni_acceptor_deinit(struct kni_acceptor * ctx) { if (ctx != NULL && ctx->ev_listener != NULL) { evconnlistener_free(ctx->ev_listener); } if (ctx != NULL && ctx->ev_base != NULL) { event_base_free(ctx->ev_base); } if (ctx != NULL && ctx->fd_unixdomain != 0) { close(ctx->fd_unixdomain); } if (ctx != NULL) { free(ctx); } return; } struct kni_acceptor * kni_acceptor_init(struct tfe_proxy * proxy, const char * profile, void * logger) { struct kni_acceptor * __ctx = ALLOC(struct kni_acceptor, 1); struct sockaddr_un __sockaddr_un; int ret = 0; __ctx->proxy = proxy; __ctx->profile = profile; __ctx->logger = logger; /* Read the unix domain socket file, this file is used to recieve fds from KNI */ MESA_load_profile_string_def(profile, "kni", "uxdomain", __ctx->str_unixdomain_file, sizeof(__ctx->str_unixdomain_file), TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT); if (unlikely(unlink(__ctx->str_unixdomain_file) < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at unlink undomain file %s: %s", __ctx->str_unixdomain_file, strerror(errno)); goto __errout; } __sockaddr_un.sun_family = AF_UNIX; strncpy(__sockaddr_un.sun_path, __ctx->str_unixdomain_file, sizeof(__sockaddr_un.sun_path)); /* Create new event base, this event base will be dispatched at separated thread */ __ctx->ev_base = event_base_new(); if (unlikely(__ctx->ev_base == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. "); goto __errout; } /* Create a listener */ __ctx->ev_listener = evconnlistener_new_bind(__ctx->ev_base, __kni_listener_accept_cb, __ctx, 0, TFE_CONFIG_BACKLOG_DEFAULT, (struct sockaddr *) (&__sockaddr_un), sizeof(__sockaddr_un)); if (unlikely(__ctx->ev_listener == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating evconnlistener."); goto __errout; } /* Create a thread to dispatch ctx->evbase */ ret = pthread_create(&__ctx->thread, NULL, __kni_listener_thread_entry, (void *) __ctx); if (ret < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating listener thread: %s", strerror(errno)); goto __errout; } TFE_LOG_INFO(__ctx->logger, "KNI acceptor unixdomain file: %s", __ctx->str_unixdomain_file); return __ctx; __errout: kni_acceptor_deinit(__ctx); return NULL; }