实现从KNI中接收上行、下行连接FD的功能。

common/CMakeLists.txt

@@ -1,3 +1,4 @@
 
 add_library(common src/tfe_stat.cpp src/tfe_utils.cpp)
 target_include_directories(common PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
+target_link_libraries(common MESA_handle_logger)

common/include/tfe_stream.h

@@ -3,9 +3,7 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <tfe_types.h>
-
+#include <tfe_utils.h>
-#define TFE_STRING_MAX        2048
-#define TFE_SYMBOL_MAX        64
 
 enum tfe_session_proto
 {

common/include/tfe_utils.h

@@ -1,17 +1,32 @@
-//#define ALLOC(t,n) (t *)calloc(sizeof(t),(n)) 
+/*
+ * @file tfe_utils.h
+ * This file provides common usage marcos and helper functions.
+ */
 
-	/* Allocates an array of objects using malloc() */
+#pragma once
-#define ALLOC(type, number) \
+#include <MESA/MESA_handle_logger.h>
-		((type *)calloc(sizeof(type), number))
+
+#define TFE_STRING_MAX        2048
+#define TFE_SYMBOL_MAX        64
+
+#ifndef TFE_CONFIG_BACKLOG_DEFAULT
+#define TFE_CONFIG_BACKLOG_DEFAULT		20
+#endif
+
+#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
 
 #define likely(expr)    __builtin_expect((expr), 1)
 #define unlikely(expr)  __builtin_expect((expr), 0)
 
+#define TFE_LOG_ERROR(handler, fmt, ...)															\
+do { MESA_handle_runtime_log(handler, RLOG_LV_FATAL, NULL, fmt, ##__VA_ARGS__); } while(0)			\
+
+#define TFE_LOG_INFO(handler, fmt, ...)																\
+do { MESA_handle_runtime_log(handler, RLOG_LV_INFO, NULL, fmt, ##__VA_ARGS__); } while(0)			\
+
+#define TFE_LOG_DEBUG(handler, fmt, ...)															\
+do { MESA_handle_runtime_log(handler, RLOG_LV_DEBUG, NULL, fmt, ##__VA_ARGS__); } while(0)			\
+
 int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr);
 int addr_layer_to_sock(struct layer_addr * layer_addr, struct sockaddr * sock_addr);
 char* tfe_strdup(const char* s);
-
-
-#define TFE_LOG_ERROR
-#define TFE_LOG_INFO
-#define TFE_LOG_DEBUG

platform/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 add_executable(tfe src/cert.cpp src/future.cpp src/kni.cpp src/tfe_stream.cpp src/main.cpp src/proxy.cpp)
 target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)
 target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)

platform/include/internal/kni.h

@@ -1,2 +1,2 @@
-void* io_kni_init(const char* unix_domain_path, struct event_base * attach);
+void* kni_init(const char *unix_domain_path, struct event_base *attach);
 

platform/src/kni.cpp

@@ -1,5 +1,286 @@
 #include <stdio.h>
-void* io_kni_init(const char* unix_domain_path, struct event_base * attach)
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <sys/errno.h>
+
+#include <MESA/MESA_prof_load.h>
+#include <tfe_stream.h>
+#include <evutil.h>
+#include <event2/listener.h>
+#include <pthread.h>
+#include <assert.h>
+
+#ifndef TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT
+#define TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT			"/var/run/.tfe_kni_acceptor_handler"
+#endif
+
+enum KNI_TLV_TYPE
 {
+	KNI_TLV_TYPE_PRO = 0x01,
+};
+
+enum KNI_TLV_VALUE
+{
+	KNI_TLV_VALUE_HTTP = 0x01,
+	KNI_TLV_VALUE_SSL = 0x02,
+};
+
+struct kni_tlv_info
+{
+	char type;
+	short len;
+	char value;
+};
+
+struct kni_acceptor_ctx
+{
+	/* INPUT */
+	const char * profile;
+	void * logger;
+
+	/* CONFIG */
+	char str_unixdomain_file[TFE_STRING_MAX];
+
+	/* PERSIST RUNTIME RESOURCE */
+	int fd_unixdomain;
+	struct event_base * ev_base;
+	struct evconnlistener * ev_listener;
+	pthread_t thread;
+
+	/* PEER CONNECTION RESOUCE
+	 * should close by __kni_conn_close() */
+	struct event * ev_kni_conn;
+	int fd_kni_conn;
+	pid_t pid_kni_conn;
+};
+
+void __kni_conn_close(struct kni_acceptor_ctx * ctx)
+{
+	if (ctx->fd_kni_conn != 0) close(ctx->fd_kni_conn);
+	if (ctx->ev_kni_conn != NULL) event_free(ctx->ev_kni_conn);
+	if (ctx->pid_kni_conn != 0) ctx->pid_kni_conn = 0;
+}
+
+void __kni_event_cb(evutil_socket_t fd, short what, void * user)
+{
+	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)user;
+	struct cmsghdr * __cmsghdr;
+	int * __fds;
+
+	assert(__ctx != NULL && __ctx->thread == pthread_self());
+	assert(what & EV_READ);
+
+	/* We use IOVEC to recieve the fds make by KNI.
+	 * This is a kind of magic skill to share socket fds between two(or more) process.
+     * http://man7.org/tlpi/code/online/dist/sockets/scm_rights_send.c.html
+     */
+
+	constexpr static int __TRANS_FDS_MAX = 2;
+	constexpr static int __CONTROLLEN = CMSG_SPACE(__TRANS_FDS_MAX * sizeof(int));
+
+	char __buffer[512] = {0};
+	struct kni_tlv_info * __tlv_info = (struct kni_tlv_info *)(__buffer);
+
+	struct iovec __iovec[1];
+	struct msghdr __msghdr;
+
+	char __cmptr[__CONTROLLEN];
+	__iovec[0].iov_base = __buffer;
+	__iovec[0].iov_len = sizeof(__buffer);
+
+	__msghdr.msg_iov = __iovec;
+	__msghdr.msg_iovlen = 1;
+	__msghdr.msg_name = NULL;
+	__msghdr.msg_namelen = 0;
+	__msghdr.msg_control = (void *)(__cmptr);
+	__msghdr.msg_controllen = (size_t)__CONTROLLEN;
+
+	ssize_t rd = recvmsg(fd, &__msghdr, 0);
+	if (rd == -1 && (errno == EINTR || errno == EAGAIN))
+	{
+		return;
+	}
+	else if (rd < 0)
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at recving fds from KNI connection: %s", strerror(errno));
+		goto __close_kni_connection;
+	}
+	else if (rd == 0)
+	{
+		TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u", __ctx->pid_kni_conn);
+		goto __close_kni_connection;
+	}
+
+	__cmsghdr = CMSG_FIRSTHDR(&__msghdr);
+	__fds = (int *) (CMSG_DATA(__cmsghdr));
+
+	assert(__tlv_info->type == KNI_TLV_TYPE_PRO);
+
+	enum tfe_session_proto __session_proto;
+	if (__tlv_info->value == KNI_TLV_VALUE_HTTP)
+	{
+		__session_proto = SESSION_PROTO_PLAIN;
+	}
+	else if (__tlv_info->value == KNI_TLV_VALUE_SSL)
+	{
+		__session_proto = SESSION_PROTO_SSL;
+	}
+	else
+	{
+		assert(0);
+		goto __close_kni_connection;
+	}
+
+	/* Call Proxy's Callback */
+	return;
+
+__close_kni_connection:
+	__kni_conn_close(__ctx);
+
+__drop_recieved_fds:
+	evutil_closesocket(__fds[0]);
+	evutil_closesocket(__fds[1]);
+}
+
+void __kni_listener_accept_cb(struct evconnlistener * listener, evutil_socket_t fd,
+	struct sockaddr * sk_addr, int sk_len, void * user)
+{
+	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)user;
+	struct event * __event = NULL;
+
+	struct ucred __cr{};
+	socklen_t __cr_len = sizeof(struct ucred);
+
+	/* There is only one KNI process can connect to TFE.
+	 * If ev_kni_conn is not NULL, there's already a KNI connected to TFE.
+	 * We need to refuse this connection
+	 */
+
+	if (unlikely(__ctx->ev_kni_conn != NULL))
+	{
+		TFE_LOG_ERROR(__ctx->logger, "One KNI(PID = %d) has been connected to our program, "
+     	    "close this connection", __ctx->pid_kni_conn);
+		goto __close_this_connection;
+	}
+
+	/* Get Peer's PID */
+	if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, (void *)&__cr, &__cr_len) < 0)
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at getsockopt(SO_PEERCRED) for fd %d, close this connection");
+		goto __close_this_connection;
+	}
+
+	__event = event_new(__ctx->ev_base, fd, EV_READ | EV_PERSIST, __kni_event_cb, __ctx);
+	if (unlikely(__event == NULL))
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at creating event, close this connection.");
+		goto __close_this_connection;
+	}
+
+	__ctx->fd_kni_conn = fd;
+	__ctx->ev_kni_conn = __event;
+	__ctx->pid_kni_conn = __cr.pid;
+
+	TFE_LOG_INFO(__ctx->logger, "KNI connected from process %u", __ctx->pid_kni_conn);
+	return;
+
+__close_this_connection:
+	__kni_conn_close(__ctx);
+}
+
+void * __kni_listener_thread_entry(void * args)
+{
+	struct kni_acceptor_ctx * __ctx = (struct kni_acceptor_ctx *)args;
+	assert(__ctx != NULL && __ctx->thread == pthread_self());
+
+	TFE_LOG_DEBUG(__ctx->logger, "Starting KNI listener thread...");
+	event_base_dispatch(__ctx->ev_base);
+	TFE_LOG_DEBUG(__ctx->logger, "Stoping KNI listener thread...");
+	return (void *)NULL;
+}
+
+void kni_acceptor_deinit(struct kni_acceptor_ctx *ctx)
+{
+	if (ctx != NULL && ctx->ev_listener != NULL)
+	{
+		evconnlistener_free(ctx->ev_listener);
+	}
+
+	if (ctx != NULL && ctx->ev_base != NULL)
+	{
+		event_base_free(ctx->ev_base);
+	}
+
+	if (ctx != NULL && ctx->fd_unixdomain != 0)
+	{
+		close(ctx->fd_unixdomain);
+	}
+
+	if (ctx != NULL)
+	{
+		free(ctx);
+	}
+
+	return;
+}
+
+struct kni_acceptor_ctx * kni_acceptor_init(const char *profile, void *logger)
+{
+	struct kni_acceptor_ctx * __ctx = ALLOC(struct kni_acceptor_ctx, 1);
+	struct sockaddr_un __sockaddr_un;
+	int ret = 0;
+
+	__ctx->profile = profile;
+	__ctx->logger = logger;
+
+	/* Read the unix domain socket file, this file is used to recieve fds from KNI */
+	MESA_load_profile_string_def(profile, "kni", "uxdomain", __ctx->str_unixdomain_file,
+		 sizeof(__ctx->str_unixdomain_file), TFE_CONFIG_KNI_UXDOMAIN_PATH_DEFAULT);
+
+	if (unlikely(unlink(__ctx->str_unixdomain_file) < 0))
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at unlink undomain file %s: %s",
+			__ctx->str_unixdomain_file, strerror(errno)); goto __errout;
+	}
+
+	__sockaddr_un.sun_family = AF_UNIX;
+	strncpy(__sockaddr_un.sun_path, __ctx->str_unixdomain_file, sizeof(__sockaddr_un.sun_path));
+
+	/* Create new event base, this event base will be dispatched at separated thread */
+	__ctx->ev_base = event_base_new();
+	if (unlikely(__ctx->ev_base == NULL))
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at creating event_base. ");
+		goto __errout;
+	}
+
+	/* Create a listener */
+	__ctx->ev_listener = evconnlistener_new_bind(__ctx->ev_base, __kni_listener_accept_cb, __ctx, 0,
+	TFE_CONFIG_BACKLOG_DEFAULT, (struct sockaddr *)(&__sockaddr_un), sizeof(__sockaddr_un));
+
+	if (unlikely(__ctx->ev_listener == NULL))
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at creating evconnlistener.");
+		goto __errout;
+	}
+
+	/* Create a thread to dispatch ctx->evbase */
+	ret = pthread_create(&__ctx->thread, NULL, __kni_listener_thread_entry, (void *)__ctx);
+	if (ret < 0)
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Failed at creating listener thread: %s", strerror(errno));
+		goto __errout;
+	}
+
+	TFE_LOG_INFO(__ctx->logger, "KNI UNIXDOMAIN FILE: %s", __ctx->str_unixdomain_file);
+	TFE_LOG_INFO(__ctx->logger, "KNI LISTENER FD: %d", __ctx->fd_unixdomain);
+
+	return __ctx;
+
+__errout:
+	kni_acceptor_deinit(__ctx);
 	return NULL;
 }