SCE增加CPU绑核，install时部署table_info.conf

CMakeLists.txt

@@ -14,7 +14,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-	set (CMAKE_INSTALL_PREFIX "/opt/tsg/tslb" CACHE PATH "default install path" FORCE)
+	set (CMAKE_INSTALL_PREFIX "/opt/tsg/sce" CACHE PATH "default install path" FORCE)
 endif()
 
 # Global Compile Options
@@ -60,4 +60,5 @@ add_subdirectory(conf)
 add_subdirectory(vendor)
 add_subdirectory(common)
 add_subdirectory(platform)
-add_subdirectory(script)
+add_subdirectory(script)
+add_subdirectory(resource)

cmake/Package.cmake

@@ -30,4 +30,8 @@ else()
     set(CPACK_RPM_PACKAGE_CONFLICTS "sce-debug")
 endif()
 
+# setup %config(noreplace)
+set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/sce.conf"
+                            "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/zlog.conf")
+
 include(CPack)

conf/sce.conf

@@ -1,7 +1,7 @@
 [system]
 nr_worker_threads=8
-enable_cpu_affinity=0
-cpu_affinity_mask=2-10
+enable_cpu_affinity=1
+cpu_affinity_mask=2,3,4-9
 firewall_sids=1001
 
 [maat]

platform/include/sce.h

@@ -10,6 +10,8 @@ extern "C"
 #include "packet_io.h"
 #include "session_table.h"
 
+#define MAX_THREAD_NUM 128
+
 /******************************************************************************
  * Struct For Thread
  ******************************************************************************/
@@ -28,6 +30,7 @@ struct thread_ctx
 
     int session_table_need_reset;
     int sf_metrics_need_send;
+    int cpu_mask;
 };
 
 /******************************************************************************
@@ -73,11 +76,13 @@ struct sce_ctx
 {
     int firewall_sids;
     int nr_worker_threads;
+    int enable_cpu_affinity;
+    int cpu_affinity_mask[MAX_THREAD_NUM];
 
     struct packet_io *io;
     struct global_metrics *metrics;
     struct policy_enforcer *enforcer;
-    struct thread_ctx work_threads[128];
+    struct thread_ctx work_threads[MAX_THREAD_NUM];
 };
 
 struct sce_ctx *sce_ctx_create(const char *profile);

platform/src/main.cpp

@@ -2,6 +2,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include <pthread.h>
+#include <sys/prctl.h>
 
 #include "sce.h"
 #include "log.h"
@@ -19,13 +20,39 @@ static void sig_handler(int signo)
     }
 }
 
+static int thread_set_affinity(int core_id)
+{
+    int num_cores = sysconf(_SC_NPROCESSORS_ONLN);
+    if (core_id < 0 || core_id >= num_cores)
+    {
+        return EINVAL;
+    }
+
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+    CPU_SET(core_id, &cpuset);
+
+    return pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+}
+
 static void *worker_thread_cycle(void *arg)
 {
     struct thread_ctx *thread_ctx = (struct thread_ctx *)arg;
     struct packet_io *handle = thread_ctx->ref_io;
     int n_packet_recv;
 
-    LOG_INFO("%s: worker thread %d running", LOG_TAG_SCE, thread_ctx->thread_index);
+    char thread_name[16];
+    snprintf(thread_name, sizeof(thread_name), "sce:worker-%d", thread_ctx->thread_index);
+    prctl(PR_SET_NAME, (unsigned long long)thread_name, NULL, NULL, NULL);
+
+    char affinity[32] = {0};
+    if (thread_ctx->cpu_mask >= 0)
+    {
+        thread_set_affinity(thread_ctx->cpu_mask);
+        snprintf(affinity, sizeof(affinity), "affinity cpu%d", thread_ctx->cpu_mask);
+    }
+
+    LOG_INFO("%s: worker thread %d %s is running", LOG_TAG_SCE, thread_ctx->thread_index, thread_ctx->cpu_mask >= 0 ? affinity : "");
 
     while (1)
     {
@@ -96,6 +123,7 @@ int main(int argc, char **argv)
         ctx->work_threads[i].ref_sce_ctx = ctx;
         ctx->work_threads[i].session_table_need_reset = 0;
         ctx->work_threads[i].sf_metrics_need_send = 0;
+        ctx->work_threads[i].cpu_mask = ctx->enable_cpu_affinity ? ctx->cpu_affinity_mask[i] : -1;
     }
 
     for (int i = 0; i < ctx->nr_worker_threads; i++)

platform/src/sce.cpp

@@ -53,7 +53,9 @@ struct sce_ctx *sce_ctx_create(const char *profile)
 
     MESA_load_profile_int_def(profile, "system", "firewall_sids", (int *)&(ctx->firewall_sids), 1001);
     MESA_load_profile_int_def(profile, "system", "nr_worker_threads", (int *)&(ctx->nr_worker_threads), 8);
-    ctx->nr_worker_threads = MIN(ctx->nr_worker_threads, (int)(sizeof(ctx->work_threads) / sizeof(ctx->work_threads[0])));
+    MESA_load_profile_int_def(profile, "system", "enable_cpu_affinity", (int *)&ctx->enable_cpu_affinity, 0);
+    MESA_load_profile_uint_range(profile, "system", "cpu_affinity_mask", MAX_THREAD_NUM, (unsigned int *)ctx->cpu_affinity_mask);
+    ctx->nr_worker_threads = MIN(ctx->nr_worker_threads, MAX_THREAD_NUM);
 
     ctx->io = packet_io_create(profile, ctx->nr_worker_threads);
     if (ctx->io == NULL)

resource/CMakeLists.txt

@@ -0,0 +1,2 @@
+install(FILES sce.json DESTINATION resource COMPONENT Program)
+install(FILES table_info.conf DESTINATION resource COMPONENT Program)

script/service/sce.service

@@ -3,4 +3,5 @@ Description=TSG Service Chaining Engine
 After=network.target
 
 [Service]
+WorkingDirectory=/opt/tsg/sce/
 ExecStart=/opt/tsg/sce/bin/sce