diff --git a/CMakeLists.txt b/CMakeLists.txt index ed1e117..4e9e0d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set (CMAKE_INSTALL_PREFIX "/opt/tsg/tslb" CACHE PATH "default install path" FORCE) + set (CMAKE_INSTALL_PREFIX "/opt/tsg/sce" CACHE PATH "default install path" FORCE) endif() # Global Compile Options @@ -60,4 +60,5 @@ add_subdirectory(conf) add_subdirectory(vendor) add_subdirectory(common) add_subdirectory(platform) -add_subdirectory(script) \ No newline at end of file +add_subdirectory(script) +add_subdirectory(resource) \ No newline at end of file diff --git a/cmake/Package.cmake b/cmake/Package.cmake index 3f4c7bc..eef492d 100644 --- a/cmake/Package.cmake +++ b/cmake/Package.cmake @@ -30,4 +30,8 @@ else() set(CPACK_RPM_PACKAGE_CONFLICTS "sce-debug") endif() +# setup %config(noreplace) +set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/sce.conf" + "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/zlog.conf") + include(CPack) \ No newline at end of file diff --git a/conf/sce.conf b/conf/sce.conf index 68a119c..7555fee 100644 --- a/conf/sce.conf +++ b/conf/sce.conf @@ -1,7 +1,7 @@ [system] nr_worker_threads=8 -enable_cpu_affinity=0 -cpu_affinity_mask=2-10 +enable_cpu_affinity=1 +cpu_affinity_mask=2,3,4-9 firewall_sids=1001 [maat] diff --git a/platform/include/sce.h b/platform/include/sce.h index 60b1764..436390b 100644 --- a/platform/include/sce.h +++ b/platform/include/sce.h @@ -10,6 +10,8 @@ extern "C" #include "packet_io.h" #include "session_table.h" +#define MAX_THREAD_NUM 128 + /****************************************************************************** * Struct For Thread ******************************************************************************/ @@ -28,6 +30,7 @@ struct thread_ctx int session_table_need_reset; int sf_metrics_need_send; + int cpu_mask; }; /****************************************************************************** @@ -73,11 +76,13 @@ struct sce_ctx { int firewall_sids; int nr_worker_threads; + int enable_cpu_affinity; + int cpu_affinity_mask[MAX_THREAD_NUM]; struct packet_io *io; struct global_metrics *metrics; struct policy_enforcer *enforcer; - struct thread_ctx work_threads[128]; + struct thread_ctx work_threads[MAX_THREAD_NUM]; }; struct sce_ctx *sce_ctx_create(const char *profile); diff --git a/platform/src/main.cpp b/platform/src/main.cpp index 2643bb7..1516258 100644 --- a/platform/src/main.cpp +++ b/platform/src/main.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "sce.h" #include "log.h" @@ -19,13 +20,39 @@ static void sig_handler(int signo) } } +static int thread_set_affinity(int core_id) +{ + int num_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (core_id < 0 || core_id >= num_cores) + { + return EINVAL; + } + + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(core_id, &cpuset); + + return pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); +} + static void *worker_thread_cycle(void *arg) { struct thread_ctx *thread_ctx = (struct thread_ctx *)arg; struct packet_io *handle = thread_ctx->ref_io; int n_packet_recv; - LOG_INFO("%s: worker thread %d running", LOG_TAG_SCE, thread_ctx->thread_index); + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "sce:worker-%d", thread_ctx->thread_index); + prctl(PR_SET_NAME, (unsigned long long)thread_name, NULL, NULL, NULL); + + char affinity[32] = {0}; + if (thread_ctx->cpu_mask >= 0) + { + thread_set_affinity(thread_ctx->cpu_mask); + snprintf(affinity, sizeof(affinity), "affinity cpu%d", thread_ctx->cpu_mask); + } + + LOG_INFO("%s: worker thread %d %s is running", LOG_TAG_SCE, thread_ctx->thread_index, thread_ctx->cpu_mask >= 0 ? affinity : ""); while (1) { @@ -96,6 +123,7 @@ int main(int argc, char **argv) ctx->work_threads[i].ref_sce_ctx = ctx; ctx->work_threads[i].session_table_need_reset = 0; ctx->work_threads[i].sf_metrics_need_send = 0; + ctx->work_threads[i].cpu_mask = ctx->enable_cpu_affinity ? ctx->cpu_affinity_mask[i] : -1; } for (int i = 0; i < ctx->nr_worker_threads; i++) diff --git a/platform/src/sce.cpp b/platform/src/sce.cpp index 4548083..0ca48f2 100644 --- a/platform/src/sce.cpp +++ b/platform/src/sce.cpp @@ -53,7 +53,9 @@ struct sce_ctx *sce_ctx_create(const char *profile) MESA_load_profile_int_def(profile, "system", "firewall_sids", (int *)&(ctx->firewall_sids), 1001); MESA_load_profile_int_def(profile, "system", "nr_worker_threads", (int *)&(ctx->nr_worker_threads), 8); - ctx->nr_worker_threads = MIN(ctx->nr_worker_threads, (int)(sizeof(ctx->work_threads) / sizeof(ctx->work_threads[0]))); + MESA_load_profile_int_def(profile, "system", "enable_cpu_affinity", (int *)&ctx->enable_cpu_affinity, 0); + MESA_load_profile_uint_range(profile, "system", "cpu_affinity_mask", MAX_THREAD_NUM, (unsigned int *)ctx->cpu_affinity_mask); + ctx->nr_worker_threads = MIN(ctx->nr_worker_threads, MAX_THREAD_NUM); ctx->io = packet_io_create(profile, ctx->nr_worker_threads); if (ctx->io == NULL) diff --git a/resource/CMakeLists.txt b/resource/CMakeLists.txt new file mode 100644 index 0000000..ccc7488 --- /dev/null +++ b/resource/CMakeLists.txt @@ -0,0 +1,2 @@ +install(FILES sce.json DESTINATION resource COMPONENT Program) +install(FILES table_info.conf DESTINATION resource COMPONENT Program) \ No newline at end of file diff --git a/script/service/sce.service b/script/service/sce.service index b5bacb2..8f5d17e 100644 --- a/script/service/sce.service +++ b/script/service/sce.service @@ -3,4 +3,5 @@ Description=TSG Service Chaining Engine After=network.target [Service] +WorkingDirectory=/opt/tsg/sce/ ExecStart=/opt/tsg/sce/bin/sce \ No newline at end of file