gfwleak/stellar-stellar
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor stellar config

Browse Source
This commit is contained in:
luwenpeng
2024-06-25 19:12:21 +08:00
parent 276bdd1b99
commit 83bffdd008
9 changed files with 113 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
conf/stellar.toml
View File

@@ -1,6 +1,6 @@
[device] [id_generator]
base = 1 # [0, 31] snowflake_worker_id_base = 1 # [0, 31]
offset = 2 # [0, 127] snowflake_worker_id_offset = 2 # [0, 127]
[packet_io] [packet_io]
mode = dumpfile # dumpfile, marsio mode = dumpfile # dumpfile, marsio

46
src/id_generator/id_generator.cpp
View File

@@ -12,51 +12,51 @@
struct id_generator struct id_generator
{ {
uint8_t device_base; // 5bit [0, 31] uint8_t snowflake_worker_id_base; // 5bit [0, 31]
uint8_t device_offset; // 7bit [0, 127] uint8_t snowflake_worker_id_offset; // 7bit [0, 127]
uint64_t device_id; // 12bit [0, 4095] ( base << 7 | offset ) uint64_t device_id; // 12bit [0, 4095] ( base << 7 | offset )
uint64_t thread_volatile[MAX_THREAD_NUM]; uint64_t thread_volatile[MAX_THREAD_NUM];
}; };
struct id_generator global_id_generator; struct id_generator global_id_generator;
/* // return 0: success
* device_base (5bit) : [0, 31] // return -1: failed
* device_offset (7bit) : [0, 127] int id_generator_init(const struct id_generator_options *opts)
* device_id (12bit) : (base << 7 | offset)
*
* return 0: success
* return -1: failed
*/
int id_generator_init(uint8_t device_base, uint8_t device_offset)
{ {
memset(&global_id_generator, 0, sizeof(struct id_generator)); memset(&global_id_generator, 0, sizeof(struct id_generator));
if (device_base > 31) if (opts == NULL)
{ {
ID_GENERATOR_LOG_ERROR("device_base %u is invalid, range [0, 31]", device_base); ID_GENERATOR_LOG_ERROR("opts is NULL");
return -1; return -1;
} }
if (device_offset > 127) if (opts->snowflake_worker_id_base > 31)
{ {
ID_GENERATOR_LOG_ERROR("device_offset %u is invalid, range [0, 127]", device_offset); ID_GENERATOR_LOG_ERROR("snowflake_worker_id_base %u is invalid, range [0, 31]", opts->snowflake_worker_id_base);
return -1; return -1;
} }
global_id_generator.device_base = device_base; if (opts->snowflake_worker_id_offset > 127)
global_id_generator.device_offset = device_offset; {
global_id_generator.device_id = ((global_id_generator.device_base << 7) | global_id_generator.device_offset) & 0xFFF; ID_GENERATOR_LOG_ERROR("snowflake_worker_id_offset %u is invalid, range [0, 127]", opts->snowflake_worker_id_offset);
return -1;
}
ID_GENERATOR_LOG_DEBUG("device_base: %u, device_offset: %u, device_id: %u", global_id_generator.snowflake_worker_id_base = opts->snowflake_worker_id_base;
global_id_generator.device_base, global_id_generator.snowflake_worker_id_offset = opts->snowflake_worker_id_offset;
global_id_generator.device_offset, global_id_generator.device_id = ((global_id_generator.snowflake_worker_id_base << 7) | global_id_generator.snowflake_worker_id_offset) & 0xFFF;
ID_GENERATOR_LOG_DEBUG("snowflake_worker_id_base: %u, snowflake_worker_id_offset: %u, device_id: %u",
global_id_generator.snowflake_worker_id_base,
global_id_generator.snowflake_worker_id_offset,
global_id_generator.device_id); global_id_generator.device_id);
return 0; return 0;
} }
/* /*
* 高位 --- 低位 * high -> low
* *
* +------+------------------+----------------+------------------------+---------------------------+ * +------+------------------+----------------+------------------------+---------------------------+
* | 1bit | 12bit device_id | 8bit thread_id | 28bit timestamp in sec | 15bit sequence per thread | * | 1bit | 12bit device_id | 8bit thread_id | 28bit timestamp in sec | 15bit sequence per thread |

18
src/id_generator/id_generator.h
View File

@@ -7,15 +7,15 @@ extern "C"
#include <stdint.h> #include <stdint.h>
/* struct id_generator_options
* device_base (5bit) : [0, 31] {
* device_offset (7bit) : [0, 127] uint8_t snowflake_worker_id_base;
* device_id (12bit) : (base << 7 | offset) uint8_t snowflake_worker_id_offset;
* };
* return 0: success
* return -1: failed // return 0: success
*/ // return -1: failed
int id_generator_init(uint8_t device_base, uint8_t device_offset); int id_generator_init(const struct id_generator_options *opts);
uint64_t id_generator_alloc(); uint64_t id_generator_alloc();
#ifdef __cplusplus #ifdef __cplusplus

2
src/stellar/CMakeLists.txt
View File

@@ -1,4 +1,4 @@
set(SOURCE config.cpp stellar_stat.cpp stellar.cpp inject.cpp) set(SOURCE stellar_config.cpp stellar_stat.cpp stellar.cpp inject.cpp)
set(LIBRARY times plugin_manager session_manager ip_reassembly packet_io pthread fieldstat4 toml) set(LIBRARY times plugin_manager session_manager ip_reassembly packet_io pthread fieldstat4 toml)
add_library(stellar_core STATIC ${SOURCE}) add_library(stellar_core STATIC ${SOURCE})

31
src/stellar/config.h
View File

@@ -1,31 +0,0 @@
#pragma once
#ifdef __cplusplus
extern "C"
{
#endif
#include "packet_io.h"
#include "ip_reassembly.h"
#include "session_manager.h"
struct device_options
{
uint8_t base;
uint8_t offset;
};
struct stellar_config
{
struct device_options dev_opts;
struct packet_io_options io_opts;
struct ip_reassembly_options ip_opts;
struct session_manager_options sess_mgr_opts;
};
int stellar_load_config(const char *file, struct stellar_config *config);
void stellar_print_config(struct stellar_config *config);
#ifdef __cplusplus
}
#endif

28
src/stellar/stellar.cpp
View File

@@ -10,13 +10,13 @@
#include "log.h" #include "log.h"
#include "logo.h" #include "logo.h"
#include "times.h" #include "times.h"
#include "config.h"
#include "packet_def.h" #include "packet_def.h"
#include "packet_utils.h" #include "packet_utils.h"
#include "session_utils.h" #include "session_utils.h"
#include "id_generator.h" #include "id_generator.h"
#include "stellar_stat.h" #include "stellar_stat.h"
#include "stellar_utils.h" #include "stellar_utils.h"
#include "stellar_config.h"
#include "plugin_manager.h" #include "plugin_manager.h"
#define STELLAR_LOG_STATE(format, ...) LOG_STATE("stellar", format, ##__VA_ARGS__) #define STELLAR_LOG_STATE(format, ...) LOG_STATE("stellar", format, ##__VA_ARGS__)
@@ -319,7 +319,7 @@ static void signal_handler(int signo)
static int all_session_have_freed(struct stellar_runtime *runtime, struct stellar_config *config) static int all_session_have_freed(struct stellar_runtime *runtime, struct stellar_config *config)
{ {
for (int i = 0; i < config->io_opts.nr_threads; i++) for (int i = 0; i < config->pkt_io_opts.nr_threads; i++)
{ {
struct session_manager *sess_mgr = runtime->threads[i].sess_mgr; struct session_manager *sess_mgr = runtime->threads[i].sess_mgr;
struct session_manager_stat *sess_stat = session_manager_stat(sess_mgr); struct session_manager_stat *sess_stat = session_manager_stat(sess_mgr);
@@ -341,7 +341,7 @@ static int all_session_have_freed(struct stellar_runtime *runtime, struct stella
static int stellar_thread_init(struct stellar_runtime *runtime, struct stellar_config *config) static int stellar_thread_init(struct stellar_runtime *runtime, struct stellar_config *config)
{ {
uint64_t now = stellar_get_monotonic_time_msec(); uint64_t now = stellar_get_monotonic_time_msec();
for (uint16_t i = 0; i < config->io_opts.nr_threads; i++) for (uint16_t i = 0; i < config->pkt_io_opts.nr_threads; i++)
{ {
struct stellar_thread *thread = &runtime->threads[i]; struct stellar_thread *thread = &runtime->threads[i];
thread->idx = i; thread->idx = i;
@@ -353,7 +353,7 @@ static int stellar_thread_init(struct stellar_runtime *runtime, struct stellar_c
STELLAR_LOG_ERROR("unable to create session manager"); STELLAR_LOG_ERROR("unable to create session manager");
return -1; return -1;
} }
thread->ip_mgr = ip_reassembly_new(&config->ip_opts); thread->ip_mgr = ip_reassembly_new(&config->ip_reass_opts);
if (thread->ip_mgr == NULL) if (thread->ip_mgr == NULL)
{ {
STELLAR_LOG_ERROR("unable to create ip reassemble manager"); STELLAR_LOG_ERROR("unable to create ip reassemble manager");
@@ -367,7 +367,7 @@ static int stellar_thread_init(struct stellar_runtime *runtime, struct stellar_c
static void stellar_thread_clean(struct stellar_runtime *runtime, struct stellar_config *config) static void stellar_thread_clean(struct stellar_runtime *runtime, struct stellar_config *config)
{ {
for (uint16_t i = 0; i < config->io_opts.nr_threads; i++) for (uint16_t i = 0; i < config->pkt_io_opts.nr_threads; i++)
{ {
struct stellar_thread *thread = &runtime->threads[i]; struct stellar_thread *thread = &runtime->threads[i];
if (ATOMIC_READ(&thread->is_runing) == 0) if (ATOMIC_READ(&thread->is_runing) == 0)
@@ -381,7 +381,7 @@ static void stellar_thread_clean(struct stellar_runtime *runtime, struct stellar
static int stellar_thread_run(struct stellar_runtime *runtime, struct stellar_config *config) static int stellar_thread_run(struct stellar_runtime *runtime, struct stellar_config *config)
{ {
for (uint16_t i = 0; i < config->io_opts.nr_threads; i++) for (uint16_t i = 0; i < config->pkt_io_opts.nr_threads; i++)
{ {
struct stellar_thread *thread = &runtime->threads[i]; struct stellar_thread *thread = &runtime->threads[i];
if (pthread_create(&thread->tid, NULL, work_thread, (void *)thread) < 0) if (pthread_create(&thread->tid, NULL, work_thread, (void *)thread) < 0)
@@ -396,7 +396,7 @@ static int stellar_thread_run(struct stellar_runtime *runtime, struct stellar_co
static void stellar_thread_join(struct stellar_runtime *runtime, struct stellar_config *config) static void stellar_thread_join(struct stellar_runtime *runtime, struct stellar_config *config)
{ {
for (uint16_t i = 0; i < config->io_opts.nr_threads; i++) for (uint16_t i = 0; i < config->pkt_io_opts.nr_threads; i++)
{ {
struct stellar_thread *thread = &runtime->threads[i]; struct stellar_thread *thread = &runtime->threads[i];
while (ATOMIC_READ(&thread->is_runing) == 1) while (ATOMIC_READ(&thread->is_runing) == 1)
@@ -410,8 +410,8 @@ static void stellar_thread_join(struct stellar_runtime *runtime, struct stellar_
int stellar_main(int argc, char **argv) int stellar_main(int argc, char **argv)
{ {
static struct stellar st = {0}; static struct stellar st = {0};
static struct stellar_runtime *runtime = &st.runtime; struct stellar_runtime *runtime = &st.runtime;
static struct stellar_config *config = &st.config; struct stellar_config *config = &st.config;
stellar_update_time_cache(); stellar_update_time_cache();
@@ -430,20 +430,20 @@ int stellar_main(int argc, char **argv)
STELLAR_LOG_STATE("main config file : %s", main_config_file); STELLAR_LOG_STATE("main config file : %s", main_config_file);
STELLAR_LOG_STATE("plugin config file : %s", plugin_config_file); STELLAR_LOG_STATE("plugin config file : %s", plugin_config_file);
if (stellar_load_config(main_config_file, config) != 0) if (stellar_config_load(config, main_config_file) != 0)
{ {
STELLAR_LOG_ERROR("unable to load config file"); STELLAR_LOG_ERROR("unable to load config file");
goto error_out; goto error_out;
} }
stellar_print_config(config); stellar_config_print(config);
if (id_generator_init(config->dev_opts.base, config->dev_opts.offset) != 0) if (id_generator_init(&config->id_gen_opts) != 0)
{ {
STELLAR_LOG_ERROR("unable to init id generator"); STELLAR_LOG_ERROR("unable to init id generator");
goto error_out; goto error_out;
} }
runtime->stat = stellar_stat_new(config->io_opts.nr_threads); runtime->stat = stellar_stat_new(config->pkt_io_opts.nr_threads);
if (runtime->stat == NULL) if (runtime->stat == NULL)
{ {
STELLAR_LOG_ERROR("unable to create stellar stat"); STELLAR_LOG_ERROR("unable to create stellar stat");
@@ -456,7 +456,7 @@ int stellar_main(int argc, char **argv)
goto error_out; goto error_out;
} }
runtime->packet_io = packet_io_new(&config->io_opts); runtime->packet_io = packet_io_new(&config->pkt_io_opts);
if (runtime->packet_io == NULL) if (runtime->packet_io == NULL)
{ {
STELLAR_LOG_ERROR("unable to create packet io"); STELLAR_LOG_ERROR("unable to create packet io");

68
src/stellar/config.cpp → src/stellar/stellar_config.cpp
View File

@@ -4,40 +4,40 @@
#include "log.h" #include "log.h"
#include "toml.h" #include "toml.h"
#include "config.h" #include "stellar_config.h"
#define CONFIG_LOG_ERROR(format, ...) LOG_ERROR("config", format, ##__VA_ARGS__) #define CONFIG_LOG_ERROR(format, ...) LOG_ERROR("config", format, ##__VA_ARGS__)
#define CONFIG_LOG_DEBUG(format, ...) LOG_DEBUG("config", format, ##__VA_ARGS__) #define CONFIG_LOG_DEBUG(format, ...) LOG_DEBUG("config", format, ##__VA_ARGS__)
// return 0: success // return 0: success
// retuun -1: failed // retuun -1: failed
static int parse_device_section(toml_table_t *root, struct device_options *opts) static int parse_id_generator_section(toml_table_t *root, struct id_generator_options *opts)
{ {
const char *ptr; const char *ptr;
toml_table_t *table; toml_table_t *table;
table = toml_table_in(root, "device"); table = toml_table_in(root, "id_generator");
if (table == NULL) if (table == NULL)
{ {
CONFIG_LOG_ERROR("config file missing device section"); CONFIG_LOG_ERROR("config file missing id_generator section");
return -1; return -1;
} }
ptr = toml_raw_in(table, "base"); ptr = toml_raw_in(table, "snowflake_worker_id_base");
if (ptr == NULL) if (ptr == NULL)
{ {
CONFIG_LOG_ERROR("config file missing device->base"); CONFIG_LOG_ERROR("config file missing id_generator->snowflake_worker_id_base");
return -1; return -1;
} }
opts->base = atoi(ptr); opts->snowflake_worker_id_base = atoi(ptr);
ptr = toml_raw_in(table, "offset"); ptr = toml_raw_in(table, "snowflake_worker_id_offset");
if (ptr == NULL) if (ptr == NULL)
{ {
CONFIG_LOG_ERROR("config file missing device->offset"); CONFIG_LOG_ERROR("config file missing id_generator->snowflake_worker_id_offset");
return -1; return -1;
} }
opts->offset = atoi(ptr); opts->snowflake_worker_id_offset = atoi(ptr);
return 0; return 0;
} }
@@ -407,7 +407,7 @@ static int parse_session_manager_section(toml_table_t *root, struct session_mana
// return 0: success // return 0: success
// retuun -1: failed // retuun -1: failed
int stellar_load_config(const char *file, struct stellar_config *config) int stellar_config_load(struct stellar_config *config, const char *file)
{ {
int ret = -1; int ret = -1;
char errbuf[200]; char errbuf[200];
@@ -429,17 +429,17 @@ int stellar_load_config(const char *file, struct stellar_config *config)
goto error_out; goto error_out;
} }
if (parse_device_section(table, &config->dev_opts) != 0) if (parse_id_generator_section(table, &config->id_gen_opts) != 0)
{ {
goto error_out; goto error_out;
} }
if (parse_packet_io_section(table, &config->io_opts) != 0) if (parse_packet_io_section(table, &config->pkt_io_opts) != 0)
{ {
goto error_out; goto error_out;
} }
if (parse_ip_reassembly_section(table, &config->ip_opts) != 0) if (parse_ip_reassembly_section(table, &config->ip_reass_opts) != 0)
{ {
goto error_out; goto error_out;
} }
@@ -465,44 +465,44 @@ error_out:
return ret; return ret;
} }
void stellar_print_config(struct stellar_config *config) void stellar_config_print(const struct stellar_config *config)
{ {
if (config == NULL) if (config == NULL)
{ {
return; return;
} }
struct device_options *dev_opts = &config->dev_opts; const struct packet_io_options *pkt_io_opts = &config->pkt_io_opts;
struct packet_io_options *io_opts = &config->io_opts; const struct id_generator_options *id_gen_opts = &config->id_gen_opts;
struct ip_reassembly_options *ip_opts = &config->ip_opts; const struct ip_reassembly_options *ip_reass_opts = &config->ip_reass_opts;
struct session_manager_options *sess_mgr_opts = &config->sess_mgr_opts; const struct session_manager_options *sess_mgr_opts = &config->sess_mgr_opts;
// device config // id_generator config
CONFIG_LOG_DEBUG("device->base : %d", dev_opts->base); CONFIG_LOG_DEBUG("id_generator->snowflake_worker_id_base : %d", id_gen_opts->snowflake_worker_id_base);
CONFIG_LOG_DEBUG("device->offset : %d", dev_opts->offset); CONFIG_LOG_DEBUG("id_generator->snowflake_worker_id_offset : %d", id_gen_opts->snowflake_worker_id_offset);
// packet io config // packet io config
CONFIG_LOG_DEBUG("packet_io->mode : %s", io_opts->mode == PACKET_IO_DUMPFILE ? "dumpfile" : "marsio"); CONFIG_LOG_DEBUG("packet_io->mode : %s", pkt_io_opts->mode == PACKET_IO_DUMPFILE ? "dumpfile" : "marsio");
if (io_opts->mode == PACKET_IO_DUMPFILE) if (pkt_io_opts->mode == PACKET_IO_DUMPFILE)
{ {
CONFIG_LOG_DEBUG("packet_io->dumpfile_dir : %s", io_opts->dumpfile_dir); CONFIG_LOG_DEBUG("packet_io->dumpfile_dir : %s", pkt_io_opts->dumpfile_dir);
} }
else else
{ {
CONFIG_LOG_DEBUG("packet_io->app_symbol : %s", io_opts->app_symbol); CONFIG_LOG_DEBUG("packet_io->app_symbol : %s", pkt_io_opts->app_symbol);
CONFIG_LOG_DEBUG("packet_io->dev_symbol : %s", io_opts->dev_symbol); CONFIG_LOG_DEBUG("packet_io->dev_symbol : %s", pkt_io_opts->dev_symbol);
} }
CONFIG_LOG_DEBUG("packet_io->nr_threads : %d", io_opts->nr_threads); CONFIG_LOG_DEBUG("packet_io->nr_threads : %d", pkt_io_opts->nr_threads);
for (uint16_t i = 0; i < io_opts->nr_threads; i++) for (uint16_t i = 0; i < pkt_io_opts->nr_threads; i++)
{ {
CONFIG_LOG_DEBUG("packet_io->cpu_mask[%3d] : %d", i, io_opts->cpu_mask[i]); CONFIG_LOG_DEBUG("packet_io->cpu_mask[%3d] : %d", i, pkt_io_opts->cpu_mask[i]);
} }
// ip reassemble config // ip reassemble config
CONFIG_LOG_DEBUG("ip_reassembly->enable : %d", ip_opts->enable); CONFIG_LOG_DEBUG("ip_reassembly->enable : %d", ip_reass_opts->enable);
CONFIG_LOG_DEBUG("ip_reassembly->timeout : %d", ip_opts->timeout); CONFIG_LOG_DEBUG("ip_reassembly->timeout : %d", ip_reass_opts->timeout);
CONFIG_LOG_DEBUG("ip_reassembly->bucket_entries : %d", ip_opts->bucket_entries); CONFIG_LOG_DEBUG("ip_reassembly->bucket_entries : %d", ip_reass_opts->bucket_entries);
CONFIG_LOG_DEBUG("ip_reassembly->bucket_num : %d", ip_opts->bucket_num); CONFIG_LOG_DEBUG("ip_reassembly->bucket_num : %d", ip_reass_opts->bucket_num);
// session manager config -> max session number // session manager config -> max session number
CONFIG_LOG_DEBUG("session_manager->max_tcp_session_num : %ld", sess_mgr_opts->max_tcp_session_num); CONFIG_LOG_DEBUG("session_manager->max_tcp_session_num : %ld", sess_mgr_opts->max_tcp_session_num);

26
src/stellar/stellar_config.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#ifdef __cplusplus
extern "C"
{
#endif
#include "packet_io.h"
#include "id_generator.h"
#include "ip_reassembly.h"
#include "session_manager.h"
struct stellar_config
{
struct packet_io_options pkt_io_opts;
struct id_generator_options id_gen_opts;
struct ip_reassembly_options ip_reass_opts;
struct session_manager_options sess_mgr_opts;
};
int stellar_config_load(struct stellar_config *config, const char *file);
void stellar_config_print(const struct stellar_config *config);
#ifdef __cplusplus
}
#endif

6
test/packet_inject/conf/stellar.toml
View File

@@ -1,6 +1,6 @@
[device] [id_generator]
base = 1 # [0, 31] snowflake_worker_id_base = 1 # [0, 31]
offset = 2 # [0, 127] snowflake_worker_id_offset = 2 # [0, 127]
[packet_io] [packet_io]
mode = marsio # dumpfile, marsio mode = marsio # dumpfile, marsio