#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT #define TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT "/var/run/.tfe_kni_acceptor_handler" #endif enum KNI_TLV_TYPE { KNI_TLV_TYPE_PRO = 0x01, }; enum KNI_TLV_VALUE { KNI_TLV_VALUE_HTTP = 0x01, KNI_TLV_VALUE_SSL = 0x02, }; struct kni_tlv_info { char type; short len; char value; }; struct kni_acceptor { /* INPUT */ struct tfe_proxy * proxy; const char * profile; void * logger; /* CONFIG */ char str_unixdomain_file[TFE_STRING_MAX]; /* PERSIST RUNTIME RESOURCE */ int fd_unixdomain; struct event_base * ev_base; struct evconnlistener * ev_listener; pthread_t thread; /* PEER CONNECTION RESOUCE * should close by __kni_conn_close() */ struct event * ev_kni_conn; int fd_kni_conn; pid_t pid_kni_conn; }; void __kni_conn_close(struct kni_acceptor * ctx) { if (ctx->fd_kni_conn != 0) close(ctx->fd_kni_conn); if (ctx->ev_kni_conn != NULL) event_free(ctx->ev_kni_conn); if (ctx->pid_kni_conn != 0) ctx->pid_kni_conn = 0; } void __kni_event_cb(evutil_socket_t fd, short what, void * user) { struct kni_acceptor * __ctx = (struct kni_acceptor *)user; struct cmsghdr * __cmsghdr; struct tfe_proxy_accept_para __accept_para; int * __fds = NULL; assert(__ctx != NULL && __ctx->thread == pthread_self()); assert(what & EV_READ); /* We use IOVEC to recieve the fds make by KNI. * This is a kind of magic skill to share socket fds between two(or more) process. * http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html */ constexpr static int __TRANS_FDS_MAX = 2; constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int)); char __buffer[512] = {0}; struct kni_tlv_info * __tlv_info = (struct kni_tlv_info *)(__buffer); struct iovec __iovec[1]; struct msghdr __msghdr; char __cmptr[__CONTROLLEN]; __iovec[0].iov_base = __buffer; __iovec[0].iov_len = sizeof(__buffer); __msghdr.msg_iov = __iovec; __msghdr.msg_iovlen = 1; __msghdr.msg_name = NULL; __msghdr.msg_namelen = 0; __msghdr.msg_control = (void *)(__cmptr); __msghdr.msg_controllen = (size_t)__CONTROLLEN; ssize_t rd = recvmsg(fd, &__msghdr, 0); if (rd == -1 && (errno == EINTR || errno == EAGAIN)) { return; } else if (rd < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at recving fds from KNI connection: %s. ", strerror(errno)); goto __close_kni_connection; } else if (rd == 0) { TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u. ", __ctx->pid_kni_conn); goto __close_kni_connection; } __cmsghdr = CMSG_FIRSTHDR(&__msghdr); __fds = (int *) (CMSG_DATA(__cmsghdr)); assert(__tlv_info->type == KNI_TLV_TYPE_PRO); enum tfe_stream_proto __session_proto; if (__tlv_info->value == KNI_TLV_VALUE_HTTP) { __session_proto = STREAM_PROTO_PLAIN; } else if (__tlv_info->value == KNI_TLV_VALUE_SSL) { __session_proto = STREAM_PROTO_SSL; } else { assert(0); goto __close_kni_connection; } __accept_para.session_type = __session_proto; __accept_para.downstream_fd = __fds[0]; __accept_para.upstream_fd = __fds[1]; if (tfe_proxy_fds_accept(__ctx->proxy, &__accept_para) < 0) { goto __drop_recieved_fds; } return; __close_kni_connection: __kni_conn_close(__ctx); __drop_recieved_fds: evutil_closesocket(__fds[0]); evutil_closesocket(__fds[1]); } void __kni_listener_accept_cb(struct evconnlistener * listener, evutil_socket_t fd, struct sockaddr * sk_addr, int sk_len, void * user) { struct kni_acceptor * __ctx = (struct kni_acceptor *)user; struct event * __event = NULL; struct ucred __cr{}; socklen_t __cr_len = sizeof(struct ucred); int ret = 0; /* There is only one KNI process can connect to TFE. * If ev_kni_conn is not NULL, there's already a KNI connected to TFE. * We need to refuse this connection */ if (unlikely(__ctx->ev_kni_conn != NULL)) { TFE_LOG_ERROR(__ctx->logger, "One KNI(PID = %d) has been connected to our program, " "close this connection", __ctx->pid_kni_conn); goto __close_this_connection; } /* Get Peer's PID */ if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, (void *)&__cr, &__cr_len) < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at getsockopt(SO_PEERCRED) for fd %d, close this connection"); goto __close_this_connection; } __event = event_new(__ctx->ev_base, fd, EV_READ | EV_PERSIST, __kni_event_cb, __ctx); if (unlikely(__event == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event, close this connection."); goto __close_this_connection; } ret = event_add(__event, NULL); if (unlikely(ret < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at adding event to evbase, close this connection. "); goto __close_this_connection; } __ctx->fd_kni_conn = fd; __ctx->ev_kni_conn = __event; __ctx->pid_kni_conn = __cr.pid; TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u", __ctx->pid_kni_conn); return; __close_this_connection: __kni_conn_close(__ctx); } void * __kni_listener_thread_entry(void * args) { struct kni_acceptor * __ctx = (struct kni_acceptor *)args; assert(__ctx != NULL && __ctx->thread == pthread_self()); TFE_LOG_DEBUG(__ctx->logger, "Starting KNI listener thread..."); event_base_dispatch(__ctx->ev_base); TFE_LOG_DEBUG(__ctx->logger, "Stoping KNI listener thread..."); return (void *)NULL; } void kni_acceptor_deinit(struct kni_acceptor *ctx) { if (ctx != NULL && ctx->ev_listener != NULL) { evconnlistener_free(ctx->ev_listener); } if (ctx != NULL && ctx->ev_base != NULL) { event_base_free(ctx->ev_base); } if (ctx != NULL && ctx->fd_unixdomain != 0) { close(ctx->fd_unixdomain); } if (ctx != NULL) { free(ctx); } return; } struct kni_acceptor * kni_acceptor_init(struct tfe_proxy *proxy, const char *profile, void *logger) { struct kni_acceptor * __ctx = ALLOC(struct kni_acceptor, 1); struct sockaddr_un __sockaddr_un; int ret = 0; __ctx->proxy = proxy; __ctx->profile = profile; __ctx->logger = logger; /* Read the unix domain socket file, this file is used to recieve fds from KNI */ MESA_load_profile_string_def(profile, "kni", "uxdomain", __ctx->str_unixdomain_file, sizeof(__ctx->str_unixdomain_file), TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT); if (unlikely(unlink(__ctx->str_unixdomain_file) < 0)) { TFE_LOG_ERROR(__ctx->logger, "Failed at unlink undomain file %s: %s", __ctx->str_unixdomain_file, strerror(errno)); goto __errout; } __sockaddr_un.sun_family = AF_UNIX; strncpy(__sockaddr_un.sun_path, __ctx->str_unixdomain_file, sizeof(__sockaddr_un.sun_path)); /* Create new event base, this event base will be dispatched at separated thread */ __ctx->ev_base = event_base_new(); if (unlikely(__ctx->ev_base == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. "); goto __errout; } /* Create a listener */ __ctx->ev_listener = evconnlistener_new_bind(__ctx->ev_base, __kni_listener_accept_cb, __ctx, 0, TFE_CONFIG_BACKLOG_DEFAULT, (struct sockaddr *)(&__sockaddr_un), sizeof(__sockaddr_un)); if (unlikely(__ctx->ev_listener == NULL)) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating evconnlistener."); goto __errout; } /* Create a thread to dispatch ctx->evbase */ ret = pthread_create(&__ctx->thread, NULL, __kni_listener_thread_entry, (void *)__ctx); if (ret < 0) { TFE_LOG_ERROR(__ctx->logger, "Failed at creating listener thread: %s", strerror(errno)); goto __errout; } TFE_LOG_INFO(__ctx->logger, "KNI acceptor unixdomain file: %s", __ctx->str_unixdomain_file); return __ctx; __errout: kni_acceptor_deinit(__ctx); return NULL; }