增加对KNI的Watchdog实现双向保活。

platform/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_executable(tfe src/acceptor_kni_v1.cpp src/acceptor_kni_v2.cpp src/ssl_stream.cpp src/key_keeper.cpp
 	src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp
 	src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp
-	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp)
+	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp)
 
 target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)
 target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)

platform/include/internal/proxy.h

@@ -8,6 +8,7 @@
 struct ssl_mgr;
 struct key_keeper;
 struct acceptor_kni_v2;
+struct watchdog_kni;
 
 enum TFE_STAT_FIELD
 {
@@ -101,6 +102,7 @@ struct tfe_proxy
     struct acceptor_kni_v1 * kni_v1_acceptor;
     struct acceptor_kni_v2 * kni_v2_acceptor;
     struct sender_scm * scm_sender;
+    struct watchdog_kni * watchdog_kni;
 
 	/* DEBUG OPTIONS */
 	unsigned int tcp_all_passthrough;

platform/include/internal/watchdog_kni.h

@@ -0,0 +1,11 @@
+//
+// Created by luqiu on 2019/6/18.
+//
+
+#ifndef TFE_WATCHDOG_KNI_H
+#define TFE_WATCHDOG_KNI_H
+
+struct watchdog_kni;
+struct watchdog_kni * watchdog_kni_create(struct tfe_proxy * proxy, const char * profile, void * logger);
+
+#endif //TFE_WATCHDOG_KNI_H

platform/src/proxy.cpp

@@ -40,6 +40,7 @@
 #include <tcp_stream.h>
 #include <acceptor_kni_v1.h>
 #include <acceptor_kni_v2.h>
+#include <watchdog_kni.h>
 
 extern struct ssl_policy_enforcer* ssl_policy_enforcer_create(void* logger);
 extern enum ssl_stream_action ssl_policy_enforce(struct ssl_stream *upstream, void* u_para);
@@ -441,6 +442,10 @@ int main(int argc, char * argv[])
     g_default_proxy->scm_sender = sender_scm_init(main_profile, "kni", g_default_logger);
     CHECK_OR_EXIT(g_default_proxy->scm_sender != NULL, "Failed at creating scm sender, Exit.");
 
+    /* Watchdog KNI */
+    g_default_proxy->watchdog_kni = watchdog_kni_create(g_default_proxy, main_profile, g_default_logger);
+    CHECK_OR_EXIT(g_default_proxy->watchdog_kni != NULL, "Failed at creating KNI watchdog, Exit.");
+
 	/* PLUGIN INIT */
 	unsigned int plugin_iterator = 0;
 	for (struct tfe_plugin * plugin_iter = tfe_plugin_iterate(&plugin_iterator);
@@ -450,8 +455,8 @@ int main(int argc, char * argv[])
 		CHECK_OR_EXIT(ret >= 0, "Plugin %s init failed. Exit. ", plugin_iter->symbol);
 		TFE_LOG_INFO(g_default_logger, "Plugin %s initialized. ", plugin_iter->symbol);
 	}
-	//ugly here. g_business_maat is available after plugin initiate.
 
+	//ugly here. g_business_maat is available after plugin initiate.
 	g_default_proxy->ssl_ply_enforcer=ssl_policy_enforcer_create(g_default_logger);
 	ssl_manager_set_new_upstream_cb(g_default_proxy->ssl_mgr_handler, ssl_policy_enforce, g_default_proxy->ssl_ply_enforcer);
 	ret = tfe_proxy_work_thread_run(g_default_proxy);

platform/src/watchdog_kni.cpp

@@ -0,0 +1,198 @@
+
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+#include <event2/bufferevent.h>
+#include <event2/event.h>
+#include <event2/buffer.h>
+#include <unistd.h>
+
+#include <tfe_utils.h>
+#include <watchdog_kni.h>
+#include <MESA/MESA_prof_load.h>
+
+
+struct watchdog_kni
+{
+    struct tfe_proxy * proxy;
+    const char * profile;
+    void * logger;
+
+    unsigned int enable;
+    struct sockaddr_in sk_kni_watchdog;
+    int fd;
+    struct event_base * ev_base;
+    struct bufferevent * bev;
+    pthread_t pthread;
+};
+
+static int watchdog_kni_fd_create()
+{
+    int fd = socket(AF_INET, SOCK_STREAM, 0);
+    unsigned int so_keepalive = 1;
+    unsigned int tcp_keepcnt = 1;
+    unsigned int tcp_keepintvl = 1;
+    unsigned int tcp_keepidle = 1;
+
+    if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (const void *) &so_keepalive, sizeof(int)) == -1)
+    {
+        TFE_LOG_ERROR(g_default_logger, "watchdog fd setup setsockopt(SO_KEEPALIVE, %d) failed : %s",
+            so_keepalive, strerror(errno)); goto errout;
+    }
+
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, (const void *) &tcp_keepcnt, sizeof(int)) == -1)
+    {
+        TFE_LOG_ERROR(g_default_logger, "watchdog fd setup setsockopt(TCP_KEEPCNT, %d) failed: %s",
+            tcp_keepcnt, strerror(errno)); goto errout;
+    }
+
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, (const void *) &tcp_keepintvl, sizeof(int)) == -1)
+    {
+        TFE_LOG_ERROR(g_default_logger, "watchdog fd setup setsockopt(TCP_KEEPINTVL, %d) failed: %s",
+            tcp_keepintvl, strerror(errno));
+    }
+
+    if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, (const void *) &tcp_keepidle, sizeof(int)) == -1)
+    {
+        TFE_LOG_ERROR(g_default_logger, "watchdog fd setup setsockopt(TCP_KEEPIDLE, %d) failed: %s",
+            tcp_keepidle, strerror(errno));
+    }
+
+    return fd;
+
+errout:
+    if(fd > 0) close(fd);
+    return -1;
+}
+
+static void watchdog_kni_readcb(struct bufferevent *bev, void *ctx)
+{
+    struct evbuffer * evbuffer_in = bufferevent_get_input(bev);
+    evbuffer_drain(evbuffer_in, evbuffer_get_length(evbuffer_in));
+}
+
+static void watchdog_kni_eventcb(struct bufferevent *bev, short what, void *ctx)
+{
+    struct watchdog_kni * __ctx = (struct watchdog_kni *)ctx;
+    if (what & BEV_EVENT_CONNECTED)
+    {
+        TFE_LOG_INFO(__ctx->bev, "KNI watchdog connection is established.");
+        return;
+    }
+
+    if (what & (BEV_EVENT_EOF | BEV_EVENT_ERROR))
+    {
+        if (what & BEV_EVENT_EOF)
+        {
+            TFE_LOG_ERROR(__ctx->logger, "KNI watchdog connection broken, KNI is shutdown, EXIT.");
+            exit(EXIT_FAILURE);
+        }
+
+        if (what & BEV_EVENT_ERROR)
+        {
+            if (errno)
+            {
+                TFE_LOG_ERROR(__ctx->logger, "KNI watchdog connection broken: %s, EXIT.", strerror(errno));
+            }
+            else
+            {
+                TFE_LOG_ERROR(__ctx->logger, "KNI watchdog connection broken: Unknown, EXIT.");
+            }
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    return;
+}
+
+void * watchdog_kni_thread(void * arg)
+{
+    struct watchdog_kni * __ctx = (struct watchdog_kni *)arg;
+    while(event_base_dispatch(__ctx->ev_base) >= 0) {}
+    DIE("watchdog thread is terminated.");
+}
+
+struct watchdog_kni * watchdog_kni_create(struct tfe_proxy * proxy, const char * profile, void * logger)
+{
+    struct watchdog_kni * __ctx = ALLOC(struct watchdog_kni, 1);
+    int ret = 0;
+
+    __ctx->proxy = proxy;
+    __ctx->profile = profile;
+    __ctx->logger = logger;
+
+    unsigned int en_watchdog = 0;
+    MESA_load_profile_uint_def(profile, "kni", "watchdog_switch", &en_watchdog, 0);
+    __ctx->enable = en_watchdog;
+
+    if (!__ctx->enable)
+    {
+        return __ctx;
+    }
+
+    char str_kni_ip[TFE_STRING_MAX] = {0};
+    MESA_load_profile_string_def(profile, "kni", "ip", str_kni_ip, sizeof(str_kni_ip), "127.0.0.1");
+
+    struct sockaddr_in sk_kni_address{};
+    sk_kni_address.sin_family = AF_INET;
+    ret = inet_pton(AF_INET, str_kni_ip, &sk_kni_address.sin_addr);
+
+    if (ret < 0)
+    {
+        TFE_LOG_ERROR(logger, "failed at parsing kni's address, in file %s, section %s, entry %s: %s",
+            profile, "kni", "ip", str_kni_ip); goto __errout;
+    }
+
+    unsigned int kni_port;
+    MESA_load_profile_uint_def(profile, "kni", "watchdog_port", &kni_port, 2476);
+
+    __ctx->sk_kni_watchdog = sk_kni_address;
+    __ctx->sk_kni_watchdog.sin_port = htons(kni_port);
+
+    /* Prepare watchdog fd */
+    __ctx->fd = watchdog_kni_fd_create();
+    if (__ctx->fd < 0)
+    {
+        TFE_LOG_ERROR(logger, "failed at creating watchdog fd : %s", strerror(errno));
+        goto __errout;
+    }
+
+    __ctx->ev_base = event_base_new();
+    if (!__ctx->ev_base)
+    {
+        TFE_LOG_ERROR(logger, "failed at watchdog event_base_new(): %s", strerror(errno));
+        goto __errout;
+    }
+
+    __ctx->bev = bufferevent_socket_new(__ctx->ev_base, __ctx->fd, BEV_OPT_CLOSE_ON_FREE);
+    if (!__ctx->bev)
+    {
+        TFE_LOG_ERROR(logger, "failed at watchdog bufferevent_socket_new(): %s", strerror(errno));
+        goto __errout;
+    }
+
+    ret = bufferevent_socket_connect(__ctx->bev, (const sockaddr *)&__ctx->sk_kni_watchdog,
+        sizeof(__ctx->sk_kni_watchdog));
+
+    if (ret < 0)
+    {
+        TFE_LOG_ERROR(logger, "failed at watchdog connect(): %s", strerror(errno));
+        goto __errout;
+    }
+
+    bufferevent_setcb(__ctx->bev, watchdog_kni_readcb, NULL, watchdog_kni_eventcb, __ctx);
+    bufferevent_enable(__ctx->bev, EV_READ | EV_WRITE);
+
+    /* Create a thread to dispatch ctx->evbase */
+    ret = pthread_create(&__ctx->pthread, NULL, watchdog_kni_thread, (void *) __ctx);
+    if (unlikely(ret < 0))
+    {
+        TFE_LOG_ERROR(__ctx->logger, "Failed at creating watchdog thread: %s", strerror(errno));
+        goto __errout;
+    }
+
+    TFE_LOG_INFO(__ctx->logger, "KNI watchdong init successfully.");
+    return __ctx;
+
+__errout:
+    return NULL;
+}