tango-tfe/platform/src/kni.cpp

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/errno.h>

#include <MESA/MESA_prof_load.h>
#include <tfe_stream.h>
#include <evutil.h>
#include <event2/listener.h>
#include <pthread.h>
#include <assert.h>

#ifndef TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT
#define TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT			"/var/run/.tfe_kni_acceptor_handler"
#endif

enum KNI_TLV_TYPE
{
	KNI_TLV_TYPE_PRO = 0x01,
};

enum KNI_TLV_VALUE
{
	KNI_TLV_VALUE_HTTP = 0x01,
	KNI_TLV_VALUE_SSL = 0x02,
};

struct kni_tlv_info
{
	char type;
	short len;
	char value;
};

struct kni_acceptor_ctx
{
	/* INPUT */
	const char * profile;
	void * logger;

	/* CONFIG */
	char str_unixdomain_file[TFE_STRING_MAX];

	/* PERSIST RUNTIME RESOURCE */
	int fd_unixdomain;
	struct event_base * ev_base;
	struct evconnlistener * ev_listener;
	pthread_t thread;

	/* PEER CONNECTION RESOUCE
	 * should close by __kni_conn_close() */
	struct event * ev_kni_conn;
	int fd_kni_conn;
	pid_t pid_kni_conn;
};

void __kni_conn_close(struct kni_acceptor_ctx * ctx)
{
	if (ctx->fd_kni_conn != 0) close(ctx->fd_kni_conn);
	if (ctx->ev_kni_conn != NULL) event_free(ctx->ev_kni_conn);
	if (ctx->pid_kni_conn != 0) ctx->pid_kni_conn = 0;
}

void __kni_event_cb(evutil_socket_t fd, short what, void * user)
{
	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)user;
	struct cmsghdr * __cmsghdr;
	int * __fds;

	assert(__ctx != NULL && __ctx->thread == pthread_self());
	assert(what & EV_READ);

	/* We use IOVEC to recieve the fds make by KNI.
	 * This is a kind of magic skill to share socket fds between two(or more) process.
     * http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html
     */

	constexpr static int __TRANS_FDS_MAX = 2;
	constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int));

	char __buffer[512] = {0};
	struct kni_tlv_info * __tlv_info = (struct kni_tlv_info *)(__buffer);

	struct iovec __iovec[1];
	struct msghdr __msghdr;

	char __cmptr[__CONTROLLEN];
	__iovec[0].iov_base = __buffer;
	__iovec[0].iov_len = sizeof(__buffer);

	__msghdr.msg_iov = __iovec;
	__msghdr.msg_iovlen = 1;
	__msghdr.msg_name = NULL;
	__msghdr.msg_namelen = 0;
	__msghdr.msg_control = (void *)(__cmptr);
	__msghdr.msg_controllen = (size_t)__CONTROLLEN;

	ssize_t rd = recvmsg(fd, &__msghdr, 0);
	if (rd == -1 && (errno == EINTR || errno == EAGAIN))
	{
		return;
	}
	else if (rd < 0)
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at recving fds from KNI connection: %s", strerror(errno));
		goto __close_kni_connection;
	}
	else if (rd == 0)
	{
		TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u", __ctx->pid_kni_conn);
		goto __close_kni_connection;
	}

	__cmsghdr = CMSG_FIRSTHDR(&__msghdr);
	__fds = (int *) (CMSG_DATA(__cmsghdr));

	assert(__tlv_info->type == KNI_TLV_TYPE_PRO);

	enum tfe_session_proto __session_proto;
	if (__tlv_info->value == KNI_TLV_VALUE_HTTP)
	{
		__session_proto = SESSION_PROTO_PLAIN;
	}
	else if (__tlv_info->value == KNI_TLV_VALUE_SSL)
	{
		__session_proto = SESSION_PROTO_SSL;
	}
	else
	{
		assert(0);
		goto __close_kni_connection;
	}

	/* Call Proxy's Callback */
	return;

__close_kni_connection:
	__kni_conn_close(__ctx);

__drop_recieved_fds:
	evutil_closesocket(__fds[0]);
	evutil_closesocket(__fds[1]);
}

void __kni_listener_accept_cb(struct evconnlistener * listener, evutil_socket_t fd,
	struct sockaddr * sk_addr, int sk_len, void * user)
{
	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)user;
	struct event * __event = NULL;

	struct ucred __cr{};
	socklen_t __cr_len = sizeof(struct ucred);

	/* There is only one KNI process can connect to TFE.
	 * If ev_kni_conn is not NULL, there's already a KNI connected to TFE.
	 * We need to refuse this connection
	 */

	if (unlikely(__ctx->ev_kni_conn != NULL))
	{
		TFE_LOG_ERROR(__ctx->logger, "One KNI(PID = %d) has been connected to our program, "
     	    "close this connection", __ctx->pid_kni_conn);
		goto __close_this_connection;
	}

	/* Get Peer's PID */
	if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, (void *)&__cr, &__cr_len) < 0)
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at getsockopt(SO_PEERCRED) for fd %d, close this connection");
		goto __close_this_connection;
	}

	__event = event_new(__ctx->ev_base, fd, EV_READ | EV_PERSIST, __kni_event_cb, __ctx);
	if (unlikely(__event == NULL))
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at creating event, close this connection.");
		goto __close_this_connection;
	}

	__ctx->fd_kni_conn = fd;
	__ctx->ev_kni_conn = __event;
	__ctx->pid_kni_conn = __cr.pid;

	TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u", __ctx->pid_kni_conn);
	return;

__close_this_connection:
	__kni_conn_close(__ctx);
}

void * __kni_listener_thread_entry(void * args)
{
	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)args;
	assert(__ctx != NULL && __ctx->thread == pthread_self());

	TFE_LOG_DEBUG(__ctx->logger, "Starting KNI listener thread...");
	event_base_dispatch(__ctx->ev_base);
	TFE_LOG_DEBUG(__ctx->logger, "Stoping KNI listener thread...");
	return (void *)NULL;
}

void kni_acceptor_deinit(struct kni_acceptor_ctx *ctx)
{
	if (ctx != NULL && ctx->ev_listener != NULL)
	{
		evconnlistener_free(ctx->ev_listener);
	}

	if (ctx != NULL && ctx->ev_base != NULL)
	{
		event_base_free(ctx->ev_base);
	}

	if (ctx != NULL && ctx->fd_unixdomain != 0)
	{
		close(ctx->fd_unixdomain);
	}

	if (ctx != NULL)
	{
		free(ctx);
	}

	return;
}

struct kni_acceptor_ctx * kni_acceptor_init(const char *profile, void *logger)
{
	struct kni_acceptor_ctx * __ctx = ALLOC(struct kni_acceptor_ctx, 1);
	struct sockaddr_un __sockaddr_un;
	int ret = 0;

	__ctx->profile = profile;
	__ctx->logger = logger;

	/* Read the unix domain socket file, this file is used to recieve fds from KNI */
	MESA_load_profile_string_def(profile, "kni", "uxdomain", __ctx->str_unixdomain_file,
		 sizeof(__ctx->str_unixdomain_file), TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT);

	if (unlikely(unlink(__ctx->str_unixdomain_file) < 0))
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at unlink undomain file %s: %s",
			__ctx->str_unixdomain_file, strerror(errno)); goto __errout;
	}

	__sockaddr_un.sun_family = AF_UNIX;
	strncpy(__sockaddr_un.sun_path, __ctx->str_unixdomain_file, sizeof(__sockaddr_un.sun_path));

	/* Create new event base, this event base will be dispatched at separated thread */
	__ctx->ev_base = event_base_new();
	if (unlikely(__ctx->ev_base == NULL))
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. ");
		goto __errout;
	}

	/* Create a listener */
	__ctx->ev_listener = evconnlistener_new_bind(__ctx->ev_base, __kni_listener_accept_cb, __ctx, 0,
	TFE_CONFIG_BACKLOG_DEFAULT, (struct sockaddr *)(&__sockaddr_un), sizeof(__sockaddr_un));

	if (unlikely(__ctx->ev_listener == NULL))
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at creating evconnlistener.");
		goto __errout;
	}

	/* Create a thread to dispatch ctx->evbase */
	ret = pthread_create(&__ctx->thread, NULL, __kni_listener_thread_entry, (void *)__ctx);
	if (ret < 0)
	{
		TFE_LOG_ERROR(__ctx->logger, "Failed at creating listener thread: %s", strerror(errno));
		goto __errout;
	}

	TFE_LOG_INFO(__ctx->logger, "KNI UNIXDOMAIN FILE: %s", __ctx->str_unixdomain_file);
	TFE_LOG_INFO(__ctx->logger, "KNI LISTENER FD: %d", __ctx->fd_unixdomain);

	return __ctx;

__errout:
	kni_acceptor_deinit(__ctx);
	return NULL;
}