255 lines
7.5 KiB
C++
255 lines
7.5 KiB
C++
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <sys/types.h>
|
|
#include <sys/un.h>
|
|
#include <unistd.h>
|
|
#include <sys/socket.h>
|
|
#include <sys/errno.h>
|
|
#include <pthread.h>
|
|
#include <assert.h>
|
|
#include <event2/listener.h>
|
|
#include <event2/util.h>
|
|
#include <event2/bufferevent.h>
|
|
|
|
#include <MESA/MESA_prof_load.h>
|
|
#include <tfe_stream.h>
|
|
#include <kni_acceptor.h>
|
|
#include <proxy.h>
|
|
#include <platform.h>
|
|
|
|
#ifndef TFE_CONFIG_KNI_SCM_SOCKET_FILE
|
|
#define TFE_CONFIG_KNI_SCM_SOCKET_FILE "/var/run/.tfe_kmod_scm_socket"
|
|
#endif
|
|
|
|
/* The KNI and TFE communicate with each other by UNIX-based socket,
|
|
* and the protocol between them is based on TLV format(Type-Length-Value).
|
|
* The byte order for each entry in the protocol are Host-Ordered.
|
|
*
|
|
* I. Magic and Total counts of T-L-V tuples, at front of the SOCKET stream.
|
|
* II. After Magic header, the stream consist of several T-L-V tuples.
|
|
*
|
|
* Note. Magic = 0x4d5a
|
|
* Consider of the byte align problem, the minimum length of the value is 4bytes(32-bits).
|
|
*
|
|
* 0 1 2 3
|
|
* 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | Magic | Total counts of TLV |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | Type | Length |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | Value |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | Type | Length |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | Value |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
* | ....... |
|
|
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
|
*/
|
|
|
|
struct kni_acceptor
|
|
{
|
|
/* INPUT */
|
|
struct tfe_proxy * proxy;
|
|
const char * profile;
|
|
void * logger;
|
|
|
|
/* CONFIG */
|
|
char str_scm_socket_file[TFE_STRING_MAX];
|
|
|
|
/* PERSIST RUNTIME RESOURCE */
|
|
int fd_scm_socket;
|
|
struct event_base * ev_base;
|
|
struct event * ev_scm_socket;
|
|
pthread_t thread;
|
|
};
|
|
|
|
void __kni_event_cb(evutil_socket_t fd, short what, void * user)
|
|
{
|
|
struct kni_acceptor * __ctx = (struct kni_acceptor *) user;
|
|
struct cmsghdr * __cmsghdr;
|
|
struct tfe_proxy_accept_para __accept_para{};
|
|
|
|
int * __fds = NULL;
|
|
assert(__ctx != NULL && __ctx->thread == pthread_self());
|
|
assert(what & EV_READ);
|
|
|
|
/* We use IOVEC to recieve the fds make by KNI.
|
|
* This is a kind of magic skill to share socket fds between two(or more) process.
|
|
* http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html
|
|
*/
|
|
|
|
constexpr static int __TRANS_FDS_MAX = 2;
|
|
constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int));
|
|
|
|
char __buffer[4096] = {0};
|
|
struct iovec __iovec[1];
|
|
struct msghdr __msghdr;
|
|
|
|
char __cmptr[__CONTROLLEN];
|
|
|
|
__iovec[0].iov_base = __buffer;
|
|
__iovec[0].iov_len = sizeof(__buffer);
|
|
|
|
__msghdr.msg_iov = __iovec;
|
|
__msghdr.msg_iovlen = 1;
|
|
__msghdr.msg_name = NULL;
|
|
__msghdr.msg_namelen = 0;
|
|
__msghdr.msg_control = (void *) (__cmptr);
|
|
__msghdr.msg_controllen = (size_t) __CONTROLLEN;
|
|
|
|
ssize_t rd = recvmsg(fd, &__msghdr, 0);
|
|
if (rd == -1 && (errno == EINTR || errno == EAGAIN))
|
|
{
|
|
return;
|
|
}
|
|
else if (rd <= 0)
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "failed at recvmsg from scm socket: %s. ", strerror(errno));
|
|
goto __die;
|
|
}
|
|
|
|
__cmsghdr = CMSG_FIRSTHDR(&__msghdr);
|
|
if (unlikely(__cmsghdr == NULL))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_FIRSTHDR() from incoming fds.");
|
|
goto __die;
|
|
}
|
|
|
|
__fds = (int *) (CMSG_DATA(__cmsghdr));
|
|
if (unlikely(__fds == NULL))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "failed at fetch CMSG_DATA() from incoming fds.");
|
|
goto __die;
|
|
}
|
|
|
|
#if 0
|
|
if (unlikely(__kni_parse_tlv_data(__ctx, &__accept_para, __buffer, (size_t) rd) < 0))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at parsing TLV format, close KNI connection.");
|
|
goto __close_kni_connection;
|
|
}
|
|
#endif
|
|
|
|
__accept_para.downstream_fd = __fds[0];
|
|
__accept_para.upstream_fd = __fds[1];
|
|
__accept_para.session_type = STREAM_PROTO_SSL;
|
|
|
|
TFE_PROXY_STAT_INCREASE(STAT_FD_OPEN_BY_KNI_ACCEPT, 2);
|
|
if (tfe_proxy_fds_accept(__ctx->proxy, &__accept_para) < 0)
|
|
{
|
|
goto __drop_recieved_fds;
|
|
}
|
|
|
|
return;
|
|
|
|
__die:
|
|
DIE("Broken kni scm socket connection, abort.");
|
|
return;
|
|
|
|
__drop_recieved_fds:
|
|
TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 2);
|
|
if (__fds != NULL) evutil_closesocket(__fds[0]);
|
|
if (__fds != NULL) evutil_closesocket(__fds[1]);
|
|
}
|
|
|
|
void * kni_acceptor_event_thread_entry(void * args)
|
|
{
|
|
struct kni_acceptor * __ctx = (struct kni_acceptor *) args;
|
|
assert(__ctx != NULL && __ctx->thread == pthread_self());
|
|
|
|
TFE_LOG_INFO(__ctx->logger, "kni acceptor thread is running.");
|
|
event_base_dispatch(__ctx->ev_base);
|
|
DIE("kni acceptor thread is exited, abort.");
|
|
}
|
|
|
|
void kni_acceptor_deinit(struct kni_acceptor * ctx)
|
|
{
|
|
if (ctx != NULL && ctx->ev_base != NULL)
|
|
{
|
|
event_base_free(ctx->ev_base);
|
|
}
|
|
|
|
if (ctx != NULL && ctx->fd_scm_socket != 0)
|
|
{
|
|
close(ctx->fd_scm_socket);
|
|
}
|
|
|
|
if (ctx != NULL)
|
|
{
|
|
free(ctx);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
struct kni_acceptor * kni_acceptor_init(struct tfe_proxy * proxy, const char * profile, void * logger)
|
|
{
|
|
struct kni_acceptor * __ctx = ALLOC(struct kni_acceptor, 1);
|
|
struct sockaddr_un __sockaddr_un{};
|
|
int ret = 0;
|
|
|
|
__ctx->proxy = proxy;
|
|
__ctx->profile = profile;
|
|
__ctx->logger = logger;
|
|
|
|
/* Read the unix domain socket file, this file is used to recieve fds from KNI */
|
|
MESA_load_profile_string_def(profile, "kni", "scm_socket_file", __ctx->str_scm_socket_file,
|
|
sizeof(__ctx->str_scm_socket_file), TFE_CONFIG_KNI_SCM_SOCKET_FILE);
|
|
|
|
__sockaddr_un.sun_family = AF_UNIX;
|
|
strncpy(__sockaddr_un.sun_path, __ctx->str_scm_socket_file, sizeof(__sockaddr_un.sun_path));
|
|
|
|
/* Create new event base, this event base will be dispatched at separated thread */
|
|
__ctx->ev_base = event_base_new();
|
|
if (unlikely(__ctx->ev_base == NULL))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. ");
|
|
goto __errout;
|
|
}
|
|
|
|
__ctx->fd_scm_socket = socket(AF_UNIX, SOCK_DGRAM, 0);
|
|
if (unlikely(__ctx->fd_scm_socket < 0))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at create scm socket fd: %s", strerror(errno));
|
|
goto __errout;
|
|
}
|
|
|
|
ret = connect(__ctx->fd_scm_socket, (struct sockaddr *)&__sockaddr_un, sizeof(__sockaddr_un));
|
|
if (unlikely(ret < 0))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at connecting to %s: %s", __sockaddr_un.sun_path, strerror(errno));
|
|
goto __errout;
|
|
}
|
|
|
|
__ctx->ev_scm_socket = event_new(__ctx->ev_base, __ctx->fd_scm_socket, EV_READ | EV_PERSIST, __kni_event_cb, __ctx);
|
|
if (unlikely(__ctx->ev_scm_socket == NULL))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at setup READ event for scm socket fd %d.", __ctx->fd_scm_socket);
|
|
goto __errout;
|
|
}
|
|
|
|
ret = event_add(__ctx->ev_scm_socket, NULL);
|
|
if (unlikely(ret < 0))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at adding scm socket event to evbase. ");
|
|
goto __errout;
|
|
}
|
|
|
|
/* Create a thread to dispatch ctx->evbase */
|
|
ret = pthread_create(&__ctx->thread, NULL, kni_acceptor_event_thread_entry, (void *) __ctx);
|
|
if (unlikely(ret < 0))
|
|
{
|
|
TFE_LOG_ERROR(__ctx->logger, "Failed at creating event thread: %s", strerror(errno));
|
|
goto __errout;
|
|
}
|
|
|
|
TFE_LOG_INFO(__ctx->logger, "KNI scm socket file: %s", __ctx->str_scm_socket_file);
|
|
return __ctx;
|
|
|
|
__errout:
|
|
kni_acceptor_deinit(__ctx);
|
|
return NULL;
|
|
}
|