gfwleak/stellar-stellar
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rename id_generator to snowflake

Browse Source
This commit is contained in:
luwenpeng
2024-08-16 11:02:19 +08:00
parent 6fbce43afc
commit f4d437d58b
23 changed files with 177 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
conf/stellar.toml
View File

@@ -1,6 +1,6 @@
[id_generator]
snowflake_worker_id_base = 1 # [0, 31]
snowflake_worker_id_offset = 2 # [0, 127]
[snowflake]
snowflake_base = 1 # [0, 31]
snowflake_offset = 2 # [0, 127]
[packet_io]
mode = dumpfile # dumpfile, marsio

2
src/CMakeLists.txt
View File

@@ -2,7 +2,7 @@ add_subdirectory(log)
add_subdirectory(tuple)
add_subdirectory(packet)
add_subdirectory(packet_io)
add_subdirectory(id_generator)
add_subdirectory(snowflake)
add_subdirectory(ip_reassembly)
add_subdirectory(tcp_reassembly)
add_subdirectory(duplicated_packet_filter)

5
src/id_generator/CMakeLists.txt
View File

@@ -1,5 +0,0 @@
add_library(id_generator id_generator.cpp)
target_include_directories(id_generator PUBLIC ${CMAKE_CURRENT_LIST_DIR})
target_include_directories(id_generator PUBLIC ${CMAKE_SOURCE_DIR}/src/utils)
target_include_directories(id_generator PUBLIC ${CMAKE_SOURCE_DIR}/include/stellar)
target_link_libraries(id_generator stellar_core)

80
src/id_generator/id_generator.cpp
View File

@@ -1,80 +0,0 @@
#include <time.h>
#include <string.h>
#include "log.h"
#include "utils.h"
#include "stellar.h"
#include "id_generator.h"
#define ID_GENERATOR_LOG_ERROR(format, ...) LOG_ERROR("id generator", format, ##__VA_ARGS__)
#define ID_GENERATOR_LOG_DEBUG(format, ...) LOG_DEBUG("id generator", format, ##__VA_ARGS__)
struct id_generator
{
uint8_t snowflake_worker_id_base; // 5bit [0, 31]
uint8_t snowflake_worker_id_offset; // 7bit [0, 127]
uint64_t device_id; // 12bit [0, 4095] ( base << 7 | offset )
uint64_t thread_volatile[MAX_THREAD_NUM];
};
struct id_generator global_id_generator = {};
// return 0: success
// return -1: failed
int id_generator_init(const struct id_generator_options *opts)
{
memset(&global_id_generator, 0, sizeof(struct id_generator));
if (opts == NULL)
{
ID_GENERATOR_LOG_ERROR("opts is NULL");
return -1;
}
if (opts->snowflake_worker_id_base > 31)
{
ID_GENERATOR_LOG_ERROR("snowflake_worker_id_base %u is invalid, range [0, 31]", opts->snowflake_worker_id_base);
return -1;
}
if (opts->snowflake_worker_id_offset > 127)
{
ID_GENERATOR_LOG_ERROR("snowflake_worker_id_offset %u is invalid, range [0, 127]", opts->snowflake_worker_id_offset);
return -1;
}
global_id_generator.snowflake_worker_id_base = opts->snowflake_worker_id_base;
global_id_generator.snowflake_worker_id_offset = opts->snowflake_worker_id_offset;
global_id_generator.device_id = ((global_id_generator.snowflake_worker_id_base << 7) | global_id_generator.snowflake_worker_id_offset) & 0xFFF;
ID_GENERATOR_LOG_DEBUG("snowflake_worker_id_base: %u, snowflake_worker_id_offset: %u, device_id: %u",
global_id_generator.snowflake_worker_id_base,
global_id_generator.snowflake_worker_id_offset,
global_id_generator.device_id);
return 0;
}
/*
* high -> low
*
* +------+------------------+----------------+------------------------+---------------------------+
* | 1bit | 12bit device_id | 8bit thread_id | 28bit timestamp in sec | 15bit sequence per thread |
* +------+------------------+----------------+------------------------+---------------------------+
*/
uint64_t id_generator_alloc(uint64_t now_sec)
{
#define MAX_ID_PER_THREAD (32768)
#define MAX_ID_BASE_TIME (268435456L)
uint64_t thr_idx = (uint16_t)stellar_get_current_thread_index();
uint64_t global_id = 0;
uint64_t id_per_thread = (global_id_generator.thread_volatile[thr_idx]++) % MAX_ID_PER_THREAD;
uint64_t id_base_time = now_sec % MAX_ID_BASE_TIME;
global_id = (global_id_generator.device_id << 51) |
(thr_idx << 43) |
(id_base_time << 15) |
(id_per_thread);
return global_id;
}

23
src/id_generator/id_generator.h
View File

@@ -1,23 +0,0 @@
#pragma once
#ifdef __cplusplus
extern "C"
{
#endif
#include <stdint.h>
struct id_generator_options
{
uint8_t snowflake_worker_id_base;
uint8_t snowflake_worker_id_offset;
};
// return 0: success
// return -1: failed
int id_generator_init(const struct id_generator_options *opts);
uint64_t id_generator_alloc(uint64_t now_sec);
#ifdef __cplusplus
}
#endif

2
src/session/CMakeLists.txt
View File

@@ -10,6 +10,6 @@ target_include_directories(session_manager PUBLIC ${CMAKE_CURRENT_LIST_DIR})
target_include_directories(session_manager PUBLIC ${CMAKE_SOURCE_DIR}/src/stellar)
target_include_directories(session_manager PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(session_manager PUBLIC ${CMAKE_SOURCE_DIR}/src/utils)
target_link_libraries(session_manager timeout id_generator duplicated_packet_filter evicted_session_filter log tcp_reassembly)
target_link_libraries(session_manager timeout snowflake duplicated_packet_filter evicted_session_filter log tcp_reassembly)
add_subdirectory(test)

4
src/session/session_manager.cpp
View File

@@ -8,7 +8,7 @@
#include "udp_utils.h"
#include "packet_layer.h"
#include "packet_utils.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_def.h"
#include "session_utils.h"
#include "session_pool.h"
@@ -539,7 +539,7 @@ static void session_update(struct session_manager *mgr, struct session *sess, en
{
if (session_get_current_state(sess) == SESSION_STATE_INIT)
{
session_set_id(sess, id_generator_alloc(mgr->now_ms / 1000));
session_set_id(sess, snowflake_id_generate(mgr->now_ms / 1000));
session_set_tuple6(sess, key);
session_set_tuple_direction(sess, dir);

10
src/session/test/gtest_case_tcp_fast_open.cpp
View File

@@ -3,7 +3,7 @@
#include "tuple.h"
#include "packet_def.h"
#include "packet_parse.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "tcp_reassembly.h"
@@ -421,11 +421,11 @@ TEST(CASE, TCP_FAST_OPEN)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_tcp_active_to_closing.cpp
View File

@@ -6,7 +6,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -555,11 +555,11 @@ TEST(TCP_ACTIVE_TO_CLOSING, BY_S2C_HALF_CLOSED_TIMEOUT)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_tcp_init_to_opening.cpp
View File

@@ -6,7 +6,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -776,11 +776,11 @@ TEST(TCP_INIT_TO_OPENING, BY_S2C_ASMMETRIC)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_tcp_init_to_opening_to_active_to_closing_to_closed.cpp
View File

@@ -5,7 +5,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -445,11 +445,11 @@ TEST(TCP_INIT_TO_OPENING_TO_ACTIVE_TO_CLOSING_TO_CLOSED, TEST)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_tcp_opening_to_active.cpp
View File

@@ -5,7 +5,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -232,11 +232,11 @@ TEST(TCP_OPENING_TO_ACTIVE, BY_SYNACK_S2C_DATA)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_tcp_opening_to_closing.cpp
View File

@@ -6,7 +6,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -773,11 +773,11 @@ TEST(TCP_OPENING_TO_CLOSING, BY_S2C_HALF_FIN)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_udp_init_to_opening_to_active_to_closing.cpp
View File

@@ -5,7 +5,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -157,11 +157,11 @@ TEST(UDP_INIT_TO_OPENING_TO_ACTIVE_TO_CLOSING, TEST)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

10
src/session/test/gtest_state_udp_init_to_opening_to_closing.cpp
View File

@@ -5,7 +5,7 @@
#include "packet_def.h"
#include "packet_parse.h"
#include "packet_layer.h"
#include "id_generator.h"
#include "snowflake.h"
#include "session_utils.h"
#include "session_manager.h"
#include "test_packets.h"
@@ -212,11 +212,11 @@ TEST(UDP_INIT_TO_OPENING_TO_CLOSING, BY_S2C)
int main(int argc, char **argv)
{
struct id_generator_options opt = {
.snowflake_worker_id_base = 1,
.snowflake_worker_id_offset = 2,
struct snowflake_options opt = {
.snowflake_base = 1,
.snowflake_offset = 2,
};
id_generator_init(&opt);
snowflake_id_init(&opt);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

5
src/snowflake/CMakeLists.txt Normal file
View File

@@ -0,0 +1,5 @@
add_library(snowflake snowflake.cpp)
target_include_directories(snowflake PUBLIC ${CMAKE_CURRENT_LIST_DIR})
target_include_directories(snowflake PUBLIC ${CMAKE_SOURCE_DIR}/src/utils)
target_include_directories(snowflake PUBLIC ${CMAKE_SOURCE_DIR}/include/stellar)
target_link_libraries(snowflake stellar_core)

80
src/snowflake/snowflake.cpp Normal file
View File

@@ -0,0 +1,80 @@
#include <time.h>
#include <string.h>
#include "log.h"
#include "utils.h"
#include "stellar.h"
#include "snowflake.h"
#define snowflake_LOG_ERROR(format, ...) LOG_ERROR("snowflake", format, ##__VA_ARGS__)
#define snowflake_LOG_DEBUG(format, ...) LOG_DEBUG("snowflake", format, ##__VA_ARGS__)
struct snowflake
{
uint8_t snowflake_base; // 5bit [0, 31]
uint8_t snowflake_offset; // 7bit [0, 127]
uint64_t device_id; // 12bit [0, 4095] ( base << 7 | offset )
uint64_t thread[MAX_THREAD_NUM];
};
struct snowflake g_snowflake = {};
// return 0: success
// return -1: failed
int snowflake_id_init(const struct snowflake_options *opts)
{
memset(&g_snowflake, 0, sizeof(struct snowflake));
if (opts == NULL)
{
snowflake_LOG_ERROR("opts is NULL");
return -1;
}
if (opts->snowflake_base > 31)
{
snowflake_LOG_ERROR("snowflake_base %u is invalid, range [0, 31]", opts->snowflake_base);
return -1;
}
if (opts->snowflake_offset > 127)
{
snowflake_LOG_ERROR("snowflake_offset %u is invalid, range [0, 127]", opts->snowflake_offset);
return -1;
}
g_snowflake.snowflake_base = opts->snowflake_base;
g_snowflake.snowflake_offset = opts->snowflake_offset;
g_snowflake.device_id = ((g_snowflake.snowflake_base << 7) | g_snowflake.snowflake_offset) & 0xFFF;
snowflake_LOG_DEBUG("snowflake_base: %u, snowflake_offset: %u, device_id: %u",
g_snowflake.snowflake_base,
g_snowflake.snowflake_offset,
g_snowflake.device_id);
return 0;
}
/*
* high -> low
*
* +------+------------------+----------------+------------------------+---------------------------+
* | 1bit | 12bit device_id | 8bit thread_id | 28bit timestamp in sec | 15bit sequence per thread |
* +------+------------------+----------------+------------------------+---------------------------+
*/
uint64_t snowflake_id_generate(uint64_t now_sec)
{
#define MAX_ID_PER_THREAD (32768)
#define MAX_ID_BASE_TIME (268435456L)
uint64_t thr_idx = (uint16_t)stellar_get_current_thread_index();
uint64_t global_id = 0;
uint64_t id_per_thread = (g_snowflake.thread[thr_idx]++) % MAX_ID_PER_THREAD;
uint64_t id_base_time = now_sec % MAX_ID_BASE_TIME;
global_id = (g_snowflake.device_id << 51) |
(thr_idx << 43) |
(id_base_time << 15) |
(id_per_thread);
return global_id;
}

23
src/snowflake/snowflake.h Normal file
View File

@@ -0,0 +1,23 @@
#pragma once
#ifdef __cplusplus
extern "C"
{
#endif
#include <stdint.h>
struct snowflake_options
{
uint8_t snowflake_base;
uint8_t snowflake_offset;
};
// return 0: success
// return -1: failed
int snowflake_id_init(const struct snowflake_options *opts);
uint64_t snowflake_id_generate(uint64_t now_sec);
#ifdef __cplusplus
}
#endif

28
src/stellar/stellar_config.cpp
View File

@@ -11,33 +11,33 @@
// return 0: success
// retuun -1: failed
static int parse_id_generator_section(toml_table_t *root, struct id_generator_options *opts)
static int parse_snowflake_section(toml_table_t *root, struct snowflake_options *opts)
{
const char *ptr;
toml_table_t *table;
table = toml_table_in(root, "id_generator");
table = toml_table_in(root, "snowflake");
if (table == NULL)
{
CONFIG_LOG_ERROR("config file missing id_generator section");
CONFIG_LOG_ERROR("config file missing snowflake section");
return -1;
}
ptr = toml_raw_in(table, "snowflake_worker_id_base");
ptr = toml_raw_in(table, "snowflake_base");
if (ptr == NULL)
{
CONFIG_LOG_ERROR("config file missing id_generator->snowflake_worker_id_base");
CONFIG_LOG_ERROR("config file missing snowflake->snowflake_base");
return -1;
}
opts->snowflake_worker_id_base = atoi(ptr);
opts->snowflake_base = atoi(ptr);
ptr = toml_raw_in(table, "snowflake_worker_id_offset");
ptr = toml_raw_in(table, "snowflake_offset");
if (ptr == NULL)
{
CONFIG_LOG_ERROR("config file missing id_generator->snowflake_worker_id_offset");
CONFIG_LOG_ERROR("config file missing snowflake->snowflake_offset");
return -1;
}
opts->snowflake_worker_id_offset = atoi(ptr);
opts->snowflake_offset = atoi(ptr);
return 0;
}
@@ -536,7 +536,7 @@ int stellar_config_load(struct stellar_config *config, const char *file)
goto error_out;
}
if (parse_id_generator_section(table, &config->id_gen_opts) != 0)
if (parse_snowflake_section(table, &config->snowflake_opts) != 0)
{
goto error_out;
}
@@ -585,13 +585,13 @@ void stellar_config_print(const struct stellar_config *config)
}
const struct packet_io_options *pkt_io_opts = &config->pkt_io_opts;
const struct id_generator_options *id_gen_opts = &config->id_gen_opts;
const struct snowflake_options *snowflake_opts = &config->snowflake_opts;
const struct ip_reassembly_options *ip_reass_opts = &config->ip_reass_opts;
const struct session_manager_options *sess_mgr_opts = &config->sess_mgr_opts;
// id_generator config
CONFIG_LOG_DEBUG("id_generator->snowflake_worker_id_base : %d", id_gen_opts->snowflake_worker_id_base);
CONFIG_LOG_DEBUG("id_generator->snowflake_worker_id_offset : %d", id_gen_opts->snowflake_worker_id_offset);
// snowflake config
CONFIG_LOG_DEBUG("snowflake->snowflake_base : %d", snowflake_opts->snowflake_base);
CONFIG_LOG_DEBUG("snowflake->snowflake_offset : %d", snowflake_opts->snowflake_offset);
// packet io config
CONFIG_LOG_DEBUG("packet_io->mode : %s", pkt_io_opts->mode == PACKET_IO_DUMPFILE ? "dumpfile" : "marsio");

4
src/stellar/stellar_config.h
View File

@@ -6,7 +6,7 @@ extern "C"
#endif
#include "packet_io.h"
#include "id_generator.h"
#include "snowflake.h"
#include "ip_reassembly.h"
#include "session_manager.h"
@@ -29,7 +29,7 @@ struct schedule_options
struct stellar_config
{
struct packet_io_options pkt_io_opts;
struct id_generator_options id_gen_opts;
struct snowflake_options snowflake_opts;
struct ip_reassembly_options ip_reass_opts;
struct session_manager_options sess_mgr_opts;
struct schedule_options sched_opts;

4
src/stellar/stellar_core.cpp
View File

@@ -15,7 +15,7 @@
#include "packet_def.h"
#include "packet_utils.h"
#include "session_utils.h"
#include "id_generator.h"
#include "snowflake.h"
#include "stellar_stat.h"
#include "stellar_core.h"
#include "stellar_config.h"
@@ -511,7 +511,7 @@ int stellar_run(int argc __attribute__((unused)), char **argv __attribute__((unu
}
stellar_config_print(config);
if (id_generator_init(&config->id_gen_opts) != 0)
if (snowflake_id_init(&config->snowflake_opts) != 0)
{
STELLAR_LOG_ERROR("unable to init id generator");
goto error_out;

2
src/utils/utils.h
View File

@@ -10,7 +10,7 @@ extern "C"
#include <time.h>
#define RX_BURST_MAX 32
#define MAX_THREAD_NUM 256 // limit by id_generator
#define MAX_THREAD_NUM 256 // limit by snowflake
#define ATOMIC_INC(x) __atomic_fetch_add(x, 1, __ATOMIC_RELAXED)
#define ATOMIC_DEC(x) __atomic_fetch_sub(x, 1, __ATOMIC_RELAXED)

6
test/packet_inject/conf/stellar.toml
View File

@@ -1,6 +1,6 @@
[id_generator]
snowflake_worker_id_base = 1 # [0, 31]
snowflake_worker_id_offset = 2 # [0, 127]
[snowflake]
snowflake_base = 1 # [0, 31]
snowflake_offset = 2 # [0, 127]
[packet_io]
mode = marsio # dumpfile, marsio