#include #include "utils_internal.h" #include "packet_internal.h" #include "packet_builder.h" #include "packet_manager.h" #include "fieldstat/fieldstat_easy.h" #define PACKET_MANAGER_LOG_ERROR(format, ...) STELLAR_LOG_ERROR(__thread_local_logger, "packet manager", format, ##__VA_ARGS__) #define PACKET_MANAGER_LOG_FATAL(format, ...) STELLAR_LOG_FATAL(__thread_local_logger, "packet manager", format, ##__VA_ARGS__) #define PACKET_MANAGER_LOG_INFO(format, ...) STELLAR_LOG_INFO(__thread_local_logger, "packet manager", format, ##__VA_ARGS__) struct node { char name[64]; uint64_t interested_tag_key_bits; uint64_t interested_tag_val_bits; on_packet_callback *node_entry; struct module *mod; }; #define MAX_NODE_PER_STAGE 128 struct node_array { struct node array[MAX_NODE_PER_STAGE]; uint16_t used; }; struct packet_manager_rte { enum packet_stage curr_stage; struct packet_queue queue[PACKET_QUEUE_MAX]; void *claim_arg; on_packet_callback *claim_cb; struct packet_manager_stat stat; }; struct packet_manager { uint16_t thread_num; struct exdata_schema *ex_sche; struct node_array nodes[PACKET_STAGE_MAX]; struct packet_manager_rte *rte[MAX_THREAD_NUM]; struct fieldstat_easy *fs; int fs_idx[PKT_MGR_STAT_MAX]; }; /****************************************************************************** * utils ******************************************************************************/ uint64_t packet_manager_stat_get(struct packet_manager_stat *stat, enum pkt_mgr_stat_type type) { switch (type) { #define XX(_type, _name, _val) case _type: return stat->_val; PKT_MGR_STAT_MAP(XX) #undef XX default: return 0; } } /****************************************************************************** * packet manager rte ******************************************************************************/ static struct packet_manager_rte *packet_manager_rte_new() { struct packet_manager_rte *pkt_mgr_rte = calloc(1, sizeof(struct packet_manager_rte)); if (pkt_mgr_rte == NULL) { PACKET_MANAGER_LOG_ERROR("failed to allocate memory for packet_manager_rte"); return NULL; } for (int i = 0; i < PACKET_QUEUE_MAX; i++) { TAILQ_INIT(&pkt_mgr_rte->queue[i]); } return pkt_mgr_rte; } static void packet_manager_rte_free(struct packet_manager_rte *pkt_mgr_rte) { struct packet *pkt = NULL; if (pkt_mgr_rte) { for (int i = 0; i < PACKET_QUEUE_MAX; i++) { while ((pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[i]))) { TAILQ_REMOVE(&pkt_mgr_rte->queue[i], pkt, stage_tqe); packet_free(pkt); } } free(pkt_mgr_rte); pkt_mgr_rte = NULL; } } /****************************************************************************** * packet manager ******************************************************************************/ struct packet_manager *packet_manager_new(uint16_t thread_num) { struct packet_manager *pkt_mgr = calloc(1, sizeof(struct packet_manager)); if (pkt_mgr == NULL) { PACKET_MANAGER_LOG_ERROR("failed to allocate memory for packet_manager"); return NULL; } pkt_mgr->thread_num = thread_num; pkt_mgr->ex_sche = exdata_schema_new(); if (pkt_mgr->ex_sche == NULL) { PACKET_MANAGER_LOG_ERROR("failed to create exdata_schema"); goto error_out; } pkt_mgr->fs = fieldstat_easy_new(pkt_mgr->thread_num, "packet_manager", NULL, 0); if (pkt_mgr->fs == NULL) { PACKET_MANAGER_LOG_ERROR("failed to create fieldstat_easy"); goto error_out; } if (fieldstat_easy_enable_auto_output(pkt_mgr->fs, "metrics/packet_manager.json", 2) != 0) { PACKET_MANAGER_LOG_ERROR("failed to enable auto output for fieldstat_easy"); goto error_out; } for (int i = 0; i < PKT_MGR_STAT_MAX; i++) { pkt_mgr->fs_idx[i] = fieldstat_easy_register_counter(pkt_mgr->fs, pkt_mgr_stat_str[i]); } return pkt_mgr; error_out: packet_manager_free(pkt_mgr); return NULL; } void packet_manager_free(struct packet_manager *pkt_mgr) { if (pkt_mgr) { if (pkt_mgr->fs) { fieldstat_easy_free(pkt_mgr->fs); } exdata_schema_free(pkt_mgr->ex_sche); free(pkt_mgr); pkt_mgr = NULL; } } int packet_manager_new_packet_exdata_index(struct packet_manager *pkt_mgr, const char *name, exdata_free *func, void *arg) { assert(pkt_mgr); return exdata_schema_new_index(pkt_mgr->ex_sche, name, func, arg); } int packet_manager_register_node(struct packet_manager *pkt_mgr, const char *name, enum packet_stage stage, uint64_t interested_tag_key_bits, uint64_t interested_tag_val_bits, on_packet_callback *node_entry, struct module *mod) { assert(pkt_mgr); assert(stage < PACKET_STAGE_MAX); assert(node_entry); struct node_array *nodes = &pkt_mgr->nodes[stage]; if (nodes->used >= MAX_NODE_PER_STAGE) { PACKET_MANAGER_LOG_ERROR("exceed max node per stage %d", MAX_NODE_PER_STAGE); return -1; } struct node *node = &nodes->array[nodes->used]; strncpy(node->name, name, sizeof(node->name)); node->interested_tag_key_bits = interested_tag_key_bits; node->interested_tag_val_bits = interested_tag_val_bits; node->node_entry = node_entry; node->mod = mod; nodes->used++; return 0; } int packet_manager_init(struct packet_manager *pkt_mgr, uint16_t thread_id) { assert(pkt_mgr); assert(thread_id < pkt_mgr->thread_num); pkt_mgr->rte[thread_id] = packet_manager_rte_new(); if (pkt_mgr->rte[thread_id] == NULL) { PACKET_MANAGER_LOG_ERROR("failed to create packet_manager_rte"); return -1; } else { return 0; } } void packet_manager_clean(struct packet_manager *pkt_mgr, uint16_t thread_id) { assert(pkt_mgr); assert(thread_id < pkt_mgr->thread_num); struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; packet_manager_rte_free(pkt_mgr_rte); pkt_mgr_rte = NULL; } void packet_manager_ingress(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche)); packet_tag_clean(pkt); pkt_mgr_rte->stat.pkts_ingress++; pkt_mgr_rte->stat.queue[PACKET_STAGE_PREROUTING].pkts_in++; TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[PACKET_STAGE_PREROUTING], pkt, stage_tqe); } struct packet *packet_manager_egress(struct packet_manager *pkt_mgr, uint16_t thread_id) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; struct packet *pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[PACKET_STAGE_MAX]); if (pkt) { pkt_mgr_rte->stat.pkts_egress++; pkt_mgr_rte->stat.queue[PACKET_STAGE_MAX].pkts_out++; TAILQ_REMOVE(&pkt_mgr_rte->queue[PACKET_STAGE_MAX], pkt, stage_tqe); exdata_runtime_free((struct exdata_runtime *)packet_get_user_data(pkt)); return pkt; } else { return NULL; } } static void packet_tag_set_ip_proto(struct packet *pkt) { switch (packet_get_ip_proto(pkt)) { case IPPROTO_TCP: packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_TCP); break; case IPPROTO_UDP: packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_UDP); break; case IPPROTO_ICMP: /* fall through */ case IPPROTO_ICMPV6: packet_tag_set(pkt, PKT_TAG_KEY_IPPROTO, PKT_TAG_VAL_IPPROTO_ICMP); break; default: break; } } void packet_manager_dispatch(struct packet_manager *pkt_mgr, uint16_t thread_id) { uint64_t pkt_tag_key_bits = 0; uint64_t pkt_tag_val_bits = 0; struct packet *pkt = NULL; struct node *node = NULL; struct node_array *nodes = NULL; struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; for (int i = 0; i < PACKET_STAGE_MAX; i++) { pkt_mgr_rte->curr_stage = i; nodes = &pkt_mgr->nodes[pkt_mgr_rte->curr_stage]; while ((pkt = TAILQ_FIRST(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage]))) { packet_set_stage(pkt, pkt_mgr_rte->curr_stage); packet_tag_set_ip_proto(pkt); // schedule packet may not set ip proto tag, so we need to set it here packet_set_claim(pkt, false); pkt_mgr_rte->claim_cb = NULL; pkt_mgr_rte->claim_arg = NULL; TAILQ_REMOVE(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage], pkt, stage_tqe); pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_out++; for (uint16_t j = 0; j < nodes->used; j++) { node = &nodes->array[j]; packet_tag_get(pkt, &pkt_tag_key_bits, &pkt_tag_val_bits); // pkt_tag may be changed by previous node, so we need to get it again if ((pkt_tag_key_bits & node->interested_tag_key_bits) && (pkt_tag_val_bits & node->interested_tag_val_bits)) { node->node_entry(pkt, node->mod); } } // packet has been claimed and cannot be released if (packet_is_claim(pkt)) { if (pkt_mgr_rte->claim_cb) { struct exdata_runtime *ex_rte = packet_get_user_data(pkt); exdata_runtime_reset(ex_rte); pkt_mgr_rte->claim_cb(pkt, pkt_mgr_rte->claim_arg); } continue; } if (packet_get_action(pkt) == PACKET_ACTION_DROP) { packet_manager_free_packet(pkt_mgr, thread_id, pkt); continue; } TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[pkt_mgr_rte->curr_stage + 1], pkt, stage_tqe); pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage + 1].pkts_in++; } } pkt_mgr_rte->curr_stage = PACKET_STAGE_MAX; } int packet_manager_claim_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt, on_packet_callback *cb, void *arg) { assert(pkt_mgr); assert(thread_id < pkt_mgr->thread_num); assert(pkt); struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; if (packet_is_claim(pkt)) { PACKET_MANAGER_LOG_ERROR("packet is already claimed, cannot claim again"); return -1; } else { pkt_mgr_rte->claim_cb = cb; pkt_mgr_rte->claim_arg = arg; packet_set_claim(pkt, true); pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_claim++; return 0; } } void packet_manager_schedule_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt, enum packet_stage stage) { assert(pkt_mgr); assert(thread_id < pkt_mgr->thread_num); assert(pkt); struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; if (stage >= PACKET_STAGE_MAX) { PACKET_MANAGER_LOG_ERROR("invalid stage %d", stage); assert(0); return; } pkt_mgr_rte->stat.queue[stage].pkts_schedule++; pkt_mgr_rte->stat.queue[stage].pkts_in++; TAILQ_INSERT_TAIL(&pkt_mgr_rte->queue[stage], pkt, stage_tqe); } struct packet_manager_stat *packet_manager_get_stat(struct packet_manager *pkt_mgr, uint16_t thread_id) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; return &pkt_mgr_rte->stat; } void packet_manager_print_stat(struct packet_manager *pkt_mgr, uint16_t thread_id) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; PACKET_MANAGER_LOG_INFO("runtime: %p, pkts_ingress: %lu, pkts_egress: %lu", pkt_mgr_rte, pkt_mgr_rte->stat.pkts_ingress, pkt_mgr_rte->stat.pkts_egress); for (int i = 0; i < PACKET_QUEUE_MAX; i++) { PACKET_MANAGER_LOG_INFO("runtime: %p, %-24s stat => " "pkts_in: %lu, pkts_out: %lu, pkts_claim: %lu, pkts_schedule: %lu, pkts_drop: %lu, " "pkts_dup_succ: %lu, pkts_dup_fail: %lu, " "pkts_build_tcp_succ: %lu, pkts_build_tcp_fail: %lu, " "pkts_build_udp_succ: %lu, pkts_build_udp_fail: %lu, " "pkts_build_l3_succ: %lu, pkts_build_l3_fail: %lu", pkt_mgr_rte, packet_stage_to_str(i), pkt_mgr_rte->stat.queue[i].pkts_in, pkt_mgr_rte->stat.queue[i].pkts_out, pkt_mgr_rte->stat.queue[i].pkts_claim, pkt_mgr_rte->stat.queue[i].pkts_schedule, pkt_mgr_rte->stat.queue[i].pkts_drop, pkt_mgr_rte->stat.queue[i].pkts_dup_succ, pkt_mgr_rte->stat.queue[i].pkts_dup_fail, pkt_mgr_rte->stat.queue[i].pkts_build_tcp_succ, pkt_mgr_rte->stat.queue[i].pkts_build_tcp_fail, pkt_mgr_rte->stat.queue[i].pkts_build_udp_succ, pkt_mgr_rte->stat.queue[i].pkts_build_udp_fail, pkt_mgr_rte->stat.queue[i].pkts_build_l3_succ, pkt_mgr_rte->stat.queue[i].pkts_build_l3_fail); } } struct packet *packet_manager_build_tcp_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt, uint32_t tcp_seq, uint32_t tcp_ack, uint8_t tcp_flags, const char *tcp_options, uint16_t tcp_options_len, const char *tcp_payload, uint16_t tcp_payload_len) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; struct packet *pkt = packet_build_tcp(origin_pkt, tcp_seq, tcp_ack, tcp_flags, tcp_options, tcp_options_len, tcp_payload, tcp_payload_len); if (pkt == NULL) { pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_tcp_fail++; return NULL; } pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_tcp_succ++; packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche)); packet_tag_clean(pkt); return pkt; } struct packet *packet_manager_build_udp_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt, const char *udp_payload, uint16_t udp_payload_len) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; struct packet *pkt = packet_build_udp(origin_pkt, udp_payload, udp_payload_len); if (pkt == NULL) { pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_udp_fail++; return NULL; } pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_udp_succ++; packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche)); packet_tag_clean(pkt); return pkt; } struct packet *packet_manager_build_l3_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt, uint8_t ip_proto, const char *l3_payload, uint16_t l3_payload_len) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; struct packet *pkt = packet_build_l3(origin_pkt, ip_proto, l3_payload, l3_payload_len); if (pkt == NULL) { pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_l3_fail++; return NULL; } pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_build_l3_succ++; packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche)); packet_tag_clean(pkt); return pkt; } struct packet *packet_manager_dup_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, const struct packet *origin_pkt) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; struct packet *pkt = packet_dup(origin_pkt); if (pkt == NULL) { pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_dup_fail++; return NULL; } pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_dup_succ++; packet_set_user_data(pkt, exdata_runtime_new(pkt_mgr->ex_sche)); packet_tag_clean(pkt); return pkt; } void packet_manager_free_packet(struct packet_manager *pkt_mgr, uint16_t thread_id, struct packet *pkt) { if (pkt) { struct packet_manager_rte *pkt_mgr_rte = pkt_mgr->rte[thread_id]; pkt_mgr_rte->stat.queue[pkt_mgr_rte->curr_stage].pkts_drop++; exdata_runtime_free((struct exdata_runtime *)packet_get_user_data(pkt)); packet_free(pkt); } } /****************************************************************************** * packet manager module ******************************************************************************/ static void on_polling(struct module_manager *mod_mgr, void *args) { uint64_t now_ms = clock_get_real_time_ms(); int thread_id = module_manager_get_thread_id(mod_mgr); struct packet_manager *pkt_mgr = (struct packet_manager *)args; static __thread uint64_t last_sync_stat_ms = 0; static __thread struct packet_manager_stat pkt_mgr_last_stat = {0}; if (now_ms - last_sync_stat_ms >= SYNC_STAT_INTERVAL_MS) { struct packet_manager_stat *pkt_mgr_curr_stat = &pkt_mgr->rte[thread_id]->stat; for (int i = 0; i < PKT_MGR_STAT_MAX; i++) { uint64_t val = packet_manager_stat_get(pkt_mgr_curr_stat, i) - packet_manager_stat_get(&pkt_mgr_last_stat, i); fieldstat_easy_counter_incrby(pkt_mgr->fs, thread_id, pkt_mgr->fs_idx[i], NULL, 0, val); } pkt_mgr_last_stat = *pkt_mgr_curr_stat; last_sync_stat_ms = now_ms; } } struct packet_manager *module_to_packet_manager(struct module *mod) { assert(mod); assert(strcmp(module_get_name(mod), PACKET_MANAGER_MODULE_NAME) == 0); return (struct packet_manager *)module_get_ctx(mod); } struct module *packet_manager_on_init(struct module_manager *mod_mgr) { assert(mod_mgr); uint16_t thread_num = module_manager_get_max_thread_num(mod_mgr); struct packet_manager *pkt_mgr = packet_manager_new(thread_num); if (pkt_mgr == NULL) { return NULL; } module_manager_register_polling_node(mod_mgr, on_polling, pkt_mgr); struct module *pkt_mgr_mod = module_new(PACKET_MANAGER_MODULE_NAME, NULL); if (pkt_mgr_mod == NULL) { PACKET_MANAGER_LOG_ERROR("failed to create packet_manager"); packet_manager_free(pkt_mgr); return NULL; } module_set_ctx(pkt_mgr_mod, pkt_mgr); PACKET_MANAGER_LOG_FATAL("packet_manager init"); return pkt_mgr_mod; } void packet_manager_on_exit(struct module_manager *mod_mgr __attribute__((unused)), struct module *mod) { if (mod) { struct packet_manager *pkt_mgr = module_get_ctx(mod); packet_manager_free(pkt_mgr); module_free(mod); PACKET_MANAGER_LOG_FATAL("packet_manager exit"); } } struct module *packet_manager_on_thread_init(struct module_manager *mod_mgr __attribute__((unused)), int thread_id, struct module *mod) { struct packet_manager *pkt_mgr = module_get_ctx(mod); assert(pkt_mgr); assert(thread_id < pkt_mgr->thread_num); if (packet_manager_init(pkt_mgr, thread_id) != 0) { PACKET_MANAGER_LOG_ERROR("failed to init packet_manager_init"); return NULL; } else { return mod; } } void packet_manager_on_thread_exit(struct module_manager *mod_mgr __attribute__((unused)), int thread_id, struct module *mod) { struct packet_manager *pkt_mgr = module_get_ctx(mod); if (pkt_mgr) { assert(thread_id < pkt_mgr->thread_num); packet_manager_clean(pkt_mgr, thread_id); } }