diff --git a/common/include/kni_utils.h b/common/include/kni_utils.h index 83f7ea7..5e3754f 100644 --- a/common/include/kni_utils.h +++ b/common/include/kni_utils.h @@ -158,6 +158,12 @@ enum kni_ipv6hdr_parse_error{ KNI_IPV6HDR_PARSE_ERROR_INVALID_TYPE = -3, }; + +enum kni_deploy_mode{ + KNI_DEPLOY_MODE_TUN = 0, + KNI_DEPLOY_MODE_NORMAL = 1, +}; + int kni_addr_trans_v4(struct stream_tuple4_v4 *tuple4, char *output, int len); int kni_addr_trans_v6(struct stream_tuple4_v6 *tuple4, char *output, int len); uint16_t kni_ip_checksum(const void *buf, size_t hdr_len); diff --git a/conf/kni.conf b/conf/kni.conf index 48df930..51324ca 100644 --- a/conf/kni.conf +++ b/conf/kni.conf @@ -3,22 +3,24 @@ log_path = ./log/kni/kni.log log_level = 10 tfe_node_count = 3 manage_eth = enp7s0 +#normal or tun +deploy_mode = tun +tun_name = tun_kni +src_mac_addr = 00:0e:c6:d6:72:c1 +dst_mac_addr = fe:65:b7:03:50:bd [tfe0] enabled = 1 -mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = ens1f5 ip_addr = 192.168.100.2 [tfe1] enabled = 1 -mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = ens1f6 ip_addr = 192.168.100.3 [tfe2] enabled = 1 -mac_addr = fe:65:b7:03:50:bd dev_eth_symbol = ens1f7 ip_addr = 192.168.100.4 @@ -55,7 +57,6 @@ kafka_brokerlist = 192.168.10.52:9092 [marsio] appsym = knifw -src_mac_addr = 00:0e:c6:d6:72:c1 [kafka] queue.buffering.max.messages = 1000000 diff --git a/entry/CMakeLists.txt b/entry/CMakeLists.txt index 4916c99..8a8eb0f 100644 --- a/entry/CMakeLists.txt +++ b/entry/CMakeLists.txt @@ -1,3 +1,3 @@ -add_library(kni SHARED src/kni_entry.cpp src/kni_maat.cpp src/kni_send_logger.cpp src/tfe_mgr.cpp) +add_library(kni SHARED src/kni_entry.cpp src/kni_maat.cpp src/kni_send_logger.cpp src/tfe_mgr.cpp src/kni_tun.cpp) target_include_directories(kni PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) target_link_libraries(kni common MESA_prof_load MESA_htable MESA_field_stat maatframe marsio uuid cjson rdkafka dabloom) \ No newline at end of file diff --git a/entry/include/kni_tun.h b/entry/include/kni_tun.h new file mode 100644 index 0000000..612a489 --- /dev/null +++ b/entry/include/kni_tun.h @@ -0,0 +1,10 @@ +enum kni_tun_mode{ + KNI_TUN_MODE_BLOCK = 0, + KNI_TUN_MODE_NOBLOCK = 2, +}; + +struct kni_tun_handle; +void kni_tun_destroy(struct kni_tun_handle *tun_handle); +struct kni_tun_handle* kni_tun_init(char *tun_name, enum kni_tun_mode mode, void *logger); +int kni_tun_write(struct kni_tun_handle *handle, char *buff, uint16_t buff_len); +int kni_tun_read(struct kni_tun_handle *handle, char *buff, uint16_t buff_len); \ No newline at end of file diff --git a/entry/include/tfe_mgr.h b/entry/include/tfe_mgr.h index ff08aec..d9ab994 100644 --- a/entry/include/tfe_mgr.h +++ b/entry/include/tfe_mgr.h @@ -3,6 +3,6 @@ #define BUFF_SIZE_MAX 1024 struct tfe_mgr; -struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logger); +struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, enum kni_deploy_mode depoly_mode, void *logger); void tfe_mgr_destroy(struct tfe_mgr* mgr); int tfe_mgr_alive_node_get(struct tfe_mgr *mgr, int thread_seq); \ No newline at end of file diff --git a/entry/src/kni_entry.cpp b/entry/src/kni_entry.cpp index 46e050a..bfb8ee2 100644 --- a/entry/src/kni_entry.cpp +++ b/entry/src/kni_entry.cpp @@ -22,6 +22,7 @@ bypass: drome: pme_new_fail: destroy_pme #include #include "tfe_mgr.h" #include "dablooms.h" +#include "kni_tun.h" struct kni_handle *g_kni_handle = NULL; struct kni_field_stat_handle *g_kni_fs_handle = NULL; @@ -52,12 +53,6 @@ enum stream_error{ STREAM_ERROR_TUPLE2STM_ADD_FAIL = -10, }; -enum sendto_tfe_mode{ - SENDTO_TFE_MODE_TAP = 0, - SENDTO_TFE_MODE_NORMAL = 1, -}; - -SENDTO_TFE_MODE_TAP struct http_project{ int host_len; char host[KNI_DOMAIN_MAX]; @@ -143,14 +138,12 @@ struct tfe_enabled_node{ int tfe_id; struct mr_vdev *dev_eth_handler; struct mr_sendpath *dev_eth_sendpath; - char mac_addr[6]; }; struct kni_marsio_handle{ struct mr_instance *instance; int tfe_enabled_node_count; struct tfe_enabled_node tfe_enabled_nodes[TFE_COUNT_MAX]; - char src_mac_addr[6]; }; struct protocol_identify_result{ @@ -181,8 +174,7 @@ struct tuple2stream_htable_value{ struct kni_handle{ int http_project_id; struct kni_marsio_handle *marsio_handle; - struct kni_tap_handle *tap_handle; - enum sendto_tfe_mode _sendto_tfe_mode; + struct kni_tun_handle *tun_handle; struct kni_maat_handle *maat_handle; struct kni_send_logger *send_logger; MESA_htable_handle traceid2pme_htable; @@ -193,6 +185,9 @@ struct kni_handle{ int thread_count; int dup_traffic_switch; int dup_traffic_action; + enum kni_deploy_mode deploy_mode; + char src_mac_addr[6]; + char dst_mac_addr[6]; }; struct traceid2pme_search_cb_args{ @@ -676,7 +671,24 @@ static char* add_cmsg_to_packet(struct pme_info *pmeinfo, struct pkt_info *pktin *len = offset; return new_pkt; } - + +static int add_ether_header(void *dst_data, void *raw_data, uint16_t raw_len, addr_type_t addr_type){ + char *src_mac = g_kni_handle->src_mac_addr; + char *dst_mac = g_kni_handle->dst_mac_addr; + //ether_header[14] + struct ethhdr *ether_hdr = (struct ethhdr*)dst_data; + memcpy(ether_hdr->h_dest, dst_mac, sizeof(ether_hdr->h_dest)); + memcpy(ether_hdr->h_source, src_mac, sizeof(ether_hdr->h_source)); + if(addr_type == ADDR_TYPE_IPV6){ + ether_hdr->h_proto = htons(ETH_P_IPV6); + } + else{ + ether_hdr->h_proto = htons(ETH_P_IP); + } + memcpy((char*)dst_data + sizeof(*ether_hdr), raw_data, raw_len); + return 0; +} + static int send_to_tfe_normal_mode(char *raw_data, uint16_t raw_len, int thread_seq, int tfe_id, addr_type_t addr_type){ void *logger = g_kni_handle->local_logger; struct kni_marsio_handle *handle = g_kni_handle->marsio_handle; @@ -694,8 +706,6 @@ static int send_to_tfe_normal_mode(char *raw_data, uint16_t raw_len, int thread_ } struct mr_vdev *dev_eth_handler = handle->tfe_enabled_nodes[index].dev_eth_handler; struct mr_sendpath *dev_eth_sendpath = handle->tfe_enabled_nodes[index].dev_eth_sendpath; - char *src_mac = handle->src_mac_addr; - char *dst_mac = handle->tfe_enabled_nodes[index].mac_addr; //only send one packet, alloc_ret <= nr_send <= BURST_MAX int nr_send = 1; int alloc_ret = marsio_buff_malloc_device(dev_eth_handler, tx_buffs, nr_send, 0, thread_seq); @@ -705,26 +715,19 @@ static int send_to_tfe_normal_mode(char *raw_data, uint16_t raw_len, int thread_ return -1; } for(int i = 0; i < nr_send; i++){ - char* dst_data = marsio_buff_append(tx_buffs[i], raw_len + 14); - //ethernet_header[14] - struct ethhdr *ether_hdr = (struct ethhdr*)dst_data; - memcpy(ether_hdr->h_dest, dst_mac, sizeof(ether_hdr->h_dest)); - memcpy(ether_hdr->h_source, src_mac, sizeof(ether_hdr->h_source)); - if(addr_type == ADDR_TYPE_IPV6){ - ether_hdr->h_proto = htons(ETH_P_IPV6); - } - else{ - ether_hdr->h_proto = htons(ETH_P_IP); - } - memcpy((char*)dst_data + sizeof(*ether_hdr), raw_data, raw_len); + char* dst_data = marsio_buff_append(tx_buffs[i], raw_len + sizeof(struct ethhdr)); + add_ether_header(dst_data, raw_data, raw_len, addr_type); } marsio_send_burst(dev_eth_sendpath, thread_seq, tx_buffs, nr_send); return 0; } -static int send_to_tfe_tap_mode(char *raw_data, uint16_t raw_len){ - struct kni_tap_handle *handle = g_kni_handle->tap_handle; - int ret = kni_tap_write(handle, raw_data, raw_len); +static int send_to_tfe_tun_mode(char *raw_data, uint16_t raw_len, addr_type_t addr_type){ + struct kni_tun_handle *handle = g_kni_handle->tun_handle; + char *dst_data = ALLOC(char, KNI_MTU); + add_ether_header(dst_data, raw_data, raw_len, addr_type); + int ret = kni_tun_write(handle, dst_data, raw_len + sizeof(struct ethhdr)); + FREE(&dst_data); if(ret < 0){ return -1; } @@ -732,15 +735,13 @@ static int send_to_tfe_tap_mode(char *raw_data, uint16_t raw_len){ } static int send_to_tfe(char *raw_data, uint16_t raw_len, int thread_seq, int tfe_id, addr_type_t addr_type){ - struct kni_marsio_handle *marsio_handle = g_kni_handle->marsio_handle; - struct kni_tap_handle *tap_handle = g_kni_handle->tap_handle; - int mode = g_kni_handle->_sendto_tfe_mode; + int mode = g_kni_handle->deploy_mode; int ret; - if(mode == SENDTO_TFE_MODE_TAP){ - ret = send_to_tfe_tap_mode(tap_handle, raw_data, raw_len); + if(mode == KNI_DEPLOY_MODE_TUN){ + ret = send_to_tfe_tun_mode(raw_data, raw_len, addr_type); } else{ - ret = send_to_tfe_normal_mode(marsio_handle, raw_data, raw_len, thread_seq, tfe_id, addr_type); + ret = send_to_tfe_normal_mode(raw_data, raw_len, thread_seq, tfe_id, addr_type); } return ret; } @@ -1442,7 +1443,6 @@ int tuple2stream_htable_search(MESA_htable_handle handle, struct ethhdr *ether_h } //ipv4 else{ -<<<<<<< HEAD ret = kni_ipv4_header_parse(raw_packet, &pktinfo); if(ret < 0){ char *errmsg = kni_ipv4_errmsg_get((enum kni_ipv4hdr_parse_error)ret); @@ -1466,32 +1466,6 @@ int tuple2stream_htable_search(MESA_htable_handle handle, struct ethhdr *ether_h dir = MESA_dir_reverse(value->stream->routedir); } ret = sapp_inject_pkt(value->stream, SIO_EXCLUDE_THIS_LAYER_HDR, raw_packet, pktinfo.ip_totlen, dir); -======= - struct iphdr *raw_packet_iphdr = (struct iphdr*)raw_packet; - tot_len = ntohs(raw_packet_iphdr->tot_len); - uint16_t iphdr_len = raw_packet_iphdr->ihl * 4; - struct tcphdr *raw_packet_tcphdr = (struct tcphdr*)((char*)raw_packet_iphdr + iphdr_len); - //replay packet - replay_packet = ALLOC(char, tot_len); - memcpy(replay_packet, raw_packet, tot_len); - struct iphdr *replay_packet_iphdr = (struct iphdr*)replay_packet; - struct tcphdr *replay_packet_tcphdr = (struct tcphdr*)((char*)replay_packet_iphdr + iphdr_len); - replay_packet_iphdr->saddr = raw_packet_iphdr->daddr; - replay_packet_iphdr->daddr = raw_packet_iphdr->saddr; - replay_packet_tcphdr->source = raw_packet_tcphdr->dest; - replay_packet_tcphdr->dest = raw_packet_tcphdr->source; - replay_packet_tcphdr->seq = htonl(ntohl(raw_packet_tcphdr->ack_seq) + value->first_data_len); //seq = ack + first_data_len - replay_packet_tcphdr->ack_seq = htonl(ntohl(raw_packet_tcphdr->seq) + 1); //ack = seq + 1 - replay_packet_tcphdr->window = htons(value->window); - replay_packet_iphdr->check = 0; - replay_packet_iphdr->check = kni_ip_checksum((void*)replay_packet_iphdr, iphdr_len); - replay_packet_tcphdr->check = 0; - replay_packet_tcphdr->check = kni_tcp_checksum((void*)replay_packet_tcphdr, tot_len - iphdr_len, - replay_packet_iphdr->saddr, replay_packet_iphdr->daddr); - } - //send to tfe: thread_seq = g_iThreadNum - int ret = send_to_tfe(replay_packet, tot_len, g_iThreadNum + thread_seq, tfe_id); ->>>>>>> feature-tap-mode if(ret < 0){ KNI_LOG_ERROR(logger, "Failed at sapp_inject_pkt, stream addr = %s", key_str); FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_SAPP_INJECT_FAIL], 0, FS_OP_ADD, 1); @@ -1511,23 +1485,44 @@ int tuple2stream_htable_search(MESA_htable_handle handle, struct ethhdr *ether_h } extern "C" char kni_polling_all_entry(const struct streaminfo *stream, void** pme, int thread_seq, const void* a_packet){ + void *logger = g_kni_handle->local_logger; MESA_htable_handle tuple2stream_htable = g_kni_handle->threads_handle[thread_seq].tuple2stream_htable; - //polling tfe - for(int i = 0; i < g_kni_handle->marsio_handle->tfe_enabled_node_count; i++){ - marsio_buff_t *rx_buffs[BURST_MAX]; - int nr_burst = 1; - struct mr_vdev *dev_eth_handler = g_kni_handle->marsio_handle->tfe_enabled_nodes[i].dev_eth_handler; - //receive from tfe, nr_recv <= nr_burst <= BURST_MAX - int nr_recv = marsio_recv_burst(dev_eth_handler, thread_seq, rx_buffs, nr_burst); - if(nr_recv <= 0){ - continue; - } - for(int j = 0; j < nr_recv; j++){ - struct ethhdr *ether_hdr = (struct ethhdr*)marsio_buff_mtod(rx_buffs[i]); - tuple2stream_htable_search(tuple2stream_htable, ether_hdr, thread_seq); + int flag = 0; + //normal mode + if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_NORMAL){ + //polling tfe + for(int i = 0; i < g_kni_handle->marsio_handle->tfe_enabled_node_count; i++){ + marsio_buff_t *rx_buffs[BURST_MAX]; + int nr_burst = 1; + struct mr_vdev *dev_eth_handler = g_kni_handle->marsio_handle->tfe_enabled_nodes[i].dev_eth_handler; + //receive from tfe, nr_recv <= nr_burst <= BURST_MAX + int nr_recv = marsio_recv_burst(dev_eth_handler, thread_seq, rx_buffs, nr_burst); + if(nr_recv <= 0){ + continue; + } + for(int j = 0; j < nr_recv; j++){ + struct ethhdr *ether_hdr = (struct ethhdr*)marsio_buff_mtod(rx_buffs[i]); + tuple2stream_htable_search(tuple2stream_htable, ether_hdr, thread_seq); + flag = 1; + } } } - return 0; + //tun mode + else{ + char buff[KNI_MTU]; + int ret = kni_tun_read(g_kni_handle->tun_handle, buff, sizeof(buff)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "Failed at read from tun"); + } + else{ + if(ret > 0){ + struct ethhdr *ether_hdr = (struct ethhdr*)buff; + tuple2stream_htable_search(tuple2stream_htable, ether_hdr, thread_seq); + flag = 1; + } + } + } + return flag; } @@ -1699,7 +1694,6 @@ static struct kni_marsio_handle* kni_marsio_init(const char* profile, int tfe_no void *logger = g_kni_handle->local_logger; const char* section = "marsio"; char appsym[KNI_SYMBOL_MAX]; - char src_mac_addr_str[KNI_SYMBOL_MAX]; unsigned int opt_value = 1; int tfe_node_enabled; struct mr_instance *mr_inst = NULL; @@ -1712,13 +1706,7 @@ static struct kni_marsio_handle* kni_marsio_init(const char* profile, int tfe_no KNI_LOG_ERROR(logger, "MESA_prof_load: appsym not set, profile = %s, section = %s", profile, section); goto error_out; } - ret = MESA_load_profile_string_nodef(profile, section, "src_mac_addr", src_mac_addr_str, sizeof(src_mac_addr_str)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "MESA_prof_load: src_mac_addr not set, profile = %s, section = %s", profile, section); - goto error_out; - } - KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n appsym: %s\n src_mac_addr: %s", - section, appsym, src_mac_addr_str); + KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n appsym: %s", section, appsym); mr_inst = marsio_create(); if(mr_inst == NULL){ KNI_LOG_ERROR(logger, "Failed at create marsio instance"); @@ -1726,50 +1714,27 @@ static struct kni_marsio_handle* kni_marsio_init(const char* profile, int tfe_no } handle = ALLOC(struct kni_marsio_handle, 1); handle->instance = mr_inst; - ret = sscanf(src_mac_addr_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", - &(handle->src_mac_addr[0]), &(handle->src_mac_addr[1]), - &(handle->src_mac_addr[2]), &(handle->src_mac_addr[3]), - &(handle->src_mac_addr[4]), &(handle->src_mac_addr[5])); - if(ret != 6){ - KNI_LOG_ERROR(logger, "MESA_prof_load: src_mac_addr = invalid, ret = %d, profile = %s, section = %s", ret, profile, section); - goto error_out; - } marsio_option_set(mr_inst, MARSIO_OPT_EXIT_WHEN_ERR, &opt_value, sizeof(opt_value)); marsio_init(mr_inst, appsym); j = 0; for(int i = 0; i < tfe_node_count; i++){ //load tfe conf char _section[KNI_SYMBOL_MAX]; - char mac_addr_str[KNI_SYMBOL_MAX]; char dev_eth_symbol[KNI_SYMBOL_MAX]; snprintf(_section, sizeof(_section), "tfe%d", i); MESA_load_profile_int_def(profile, _section, "enabled", &tfe_node_enabled, 1); if(tfe_node_enabled != 1){ continue; } - int ret = MESA_load_profile_string_nodef(profile, _section, "mac_addr", mac_addr_str, sizeof(mac_addr_str)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "MESA_prof_load: mac_addr not set, profile = %s, section = %s", profile, _section); - goto error_out; - } struct tfe_enabled_node tfe_node; memset(&tfe_node, 0, sizeof(tfe_node)); - //ff:ee:dd:cc:bb:aa ---> 0xff 0xee 0xdd 0xcc 0xbb 0xaa - ret = sscanf(mac_addr_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", - &(tfe_node.mac_addr[0]), &(tfe_node.mac_addr[1]), - &(tfe_node.mac_addr[2]), &(tfe_node.mac_addr[3]), - &(tfe_node.mac_addr[4]), &(tfe_node.mac_addr[5])); - if(ret != 6){ - KNI_LOG_ERROR(logger, "MESA_prof_load: mac_addr = invalid, ret = %d, profile = %s, section = %s", ret, profile, _section); - goto error_out; - } ret = MESA_load_profile_string_nodef(profile, _section, "dev_eth_symbol", dev_eth_symbol, sizeof(dev_eth_symbol)); if(ret < 0){ KNI_LOG_ERROR(logger, "MESA_prof_load: dev_eth_symbol not set, profile = %s, section = %s", profile, _section); goto error_out; } - KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n enabled: %d, mac_addr: %s\n dev_eth_symbol: %s", - _section, tfe_node_enabled, mac_addr_str, dev_eth_symbol); + KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n enabled: %d\n dev_eth_symbol: %s", + _section, tfe_node_enabled, dev_eth_symbol); //eth_handler receive thread = thread_count, send thread = thread_count dev_eth_handler = marsio_open_device(mr_inst, dev_eth_symbol, g_kni_handle->thread_count, g_kni_handle->thread_count); if(dev_eth_handler == NULL){ @@ -1892,11 +1857,13 @@ static struct kni_field_stat_handle * fs_init(const char *profile){ fs_handle->fields[KNI_FIELD_BLOOM_SEARCH_FAIL] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "bloom_srch_F"); fs_handle->fields[KNI_FIELD_BLOOM_ADD_SUCC] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "bloom_add_S"); fs_handle->fields[KNI_FIELD_BLOOM_ADD_FAIL] = FS_register(handle, FS_STYLE_FIELD, FS_CALC_CURRENT, "bloom_add_F"); - for(int i = 0; i < g_kni_handle->marsio_handle->tfe_enabled_node_count; i++){ - int tfe_id = g_kni_handle->marsio_handle->tfe_enabled_nodes[i].tfe_id; - char tfe_status[KNI_SYMBOL_MAX] = ""; - snprintf(tfe_status, sizeof(tfe_status), "tfe%d", tfe_id); - fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + i] = FS_register(handle, FS_STYLE_STATUS, FS_CALC_CURRENT, tfe_status); + if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_NORMAL){ + for(int i = 0; i < g_kni_handle->marsio_handle->tfe_enabled_node_count; i++){ + int tfe_id = g_kni_handle->marsio_handle->tfe_enabled_nodes[i].tfe_id; + char tfe_status[KNI_SYMBOL_MAX] = ""; + snprintf(tfe_status, sizeof(tfe_status), "tfe%d", tfe_id); + fs_handle->fields[KNI_FIELD_TFE_STATUS_BASE + i] = FS_register(handle, FS_STYLE_STATUS, FS_CALC_CURRENT, tfe_status); + } } //table fs_handle->column_cnt = g_kni_handle->thread_count; @@ -1949,7 +1916,7 @@ static void tuple2stream_htable_data_free_cb(void *data){ int dup_traffic_dabloom_init(const char *profile, void *logger){ const char *section = "dup_traffic"; MESA_load_profile_int_def(profile, section, "switch", &(g_kni_handle->dup_traffic_switch), 0); - KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n switch: %d", g_kni_handle->dup_traffic_switch); + KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n switch: %d", section, g_kni_handle->dup_traffic_switch); if(g_kni_handle->dup_traffic_switch == 1){ unsigned int capacity = 0; char error_rate_str[KNI_SYMBOL_MAX]; @@ -1983,7 +1950,7 @@ extern "C" int kni_init(){ //init logger char log_path[KNI_PATH_MAX] = ""; - int tfe_node_count = 0; + int tfe_node_count = 1; char manage_eth[KNI_SYMBOL_MAX] = ""; struct kni_send_logger *send_logger = NULL; struct kni_field_stat_handle *fs_handle = NULL; @@ -2009,33 +1976,69 @@ extern "C" int kni_init(){ printf("Failed at create logger: %s", log_path); goto error_out; } + g_kni_handle = ALLOC(struct kni_handle, 1); + g_kni_handle->local_logger = local_logger; + //kni_git_log KNI_LOG_ERROR(local_logger, "----------kni version = %s-----------", kni_git_verison); - ret = MESA_load_profile_int_nodef(profile, section, "tfe_node_count", &tfe_node_count); - if(ret < 0){ - KNI_LOG_ERROR(local_logger, "MESA_prof_load: tfe_node_count not set, profile = %s, section = %s", profile, section); - goto error_out; + char deploy_mode[KNI_SYMBOL_MAX]; + ret = MESA_load_profile_string_def(profile, section, "deploy_mode", deploy_mode, sizeof(deploy_mode), "normal"); + g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_NORMAL; + if(strcmp(deploy_mode, "tun") == 0){ + g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_TUN; } - - if(tfe_node_count > TFE_COUNT_MAX){ - KNI_LOG_ERROR(local_logger, "tfe_node_count = %d, exceed the max_tfe_node_count %d", tfe_node_count, TFE_COUNT_MAX); - goto error_out; - } - if(tfe_node_count <= 0){ - KNI_LOG_ERROR(local_logger, "tfe_node_count = %d, <= 0", tfe_node_count); - goto error_out; + if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_NORMAL){ + ret = MESA_load_profile_int_nodef(profile, section, "tfe_node_count", &tfe_node_count); + if(ret < 0){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: tfe_node_count not set, profile = %s, section = %s", profile, section); + goto error_out; + } + if(tfe_node_count > TFE_COUNT_MAX){ + KNI_LOG_ERROR(local_logger, "tfe_node_count = %d, exceed the max_tfe_node_count %d", tfe_node_count, TFE_COUNT_MAX); + goto error_out; + } + if(tfe_node_count <= 0){ + KNI_LOG_ERROR(local_logger, "tfe_node_count = %d, <= 0", tfe_node_count); + goto error_out; + } } ret = MESA_load_profile_string_nodef(profile, section, "manage_eth", manage_eth, sizeof(manage_eth)); if(ret < 0){ printf("MESA_prof_load: manage_eth not set, profile = %s, section = %s", profile, section); goto error_out; } - KNI_LOG_ERROR(local_logger, "MESA_prof_load, [%s]:\n log_path: %s\n log_level: %d\n tfe_node_count: %d\n manage_eth: %s", - section, log_path, log_level, tfe_node_count, manage_eth); - g_kni_handle = ALLOC(struct kni_handle, 1); - g_kni_handle->local_logger = local_logger; - + char src_mac_addr_str[KNI_SYMBOL_MAX]; + char dst_mac_addr_str[KNI_SYMBOL_MAX]; + ret = MESA_load_profile_string_nodef(profile, section, "src_mac_addr", src_mac_addr_str, sizeof(src_mac_addr_str)); + if(ret < 0){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: src_mac_addr not set, profile = %s, section = %s", profile, section); + goto error_out; + } + ret = MESA_load_profile_string_nodef(profile, section, "dst_mac_addr", dst_mac_addr_str, sizeof(dst_mac_addr_str)); + if(ret < 0){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: dst_mac_addr not set, profile = %s, section = %s", profile, section); + goto error_out; + } + KNI_LOG_ERROR(local_logger, "MESA_prof_load, [%s]:\n log_path: %s\n log_level: %d\n tfe_node_count: %d\n manage_eth: %s\n deploy_mode: %s\n" + "src_mac_addr: %s\n dst_mac_addr: %s", section, log_path, log_level, tfe_node_count, manage_eth, deploy_mode, src_mac_addr_str, dst_mac_addr_str); + //ff:ee:dd:cc:bb:aa ---> 0xff 0xee 0xdd 0xcc 0xbb 0xaa + ret = sscanf(src_mac_addr_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", + &(g_kni_handle->src_mac_addr[0]), &(g_kni_handle->src_mac_addr[1]), + &(g_kni_handle->src_mac_addr[2]), &(g_kni_handle->src_mac_addr[3]), + &(g_kni_handle->src_mac_addr[4]), &(g_kni_handle->src_mac_addr[5])); + if(ret != 6){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: src_mac_addr = invalid, ret = %d, profile = %s, section = %s", ret, profile, section); + goto error_out; + } + ret = sscanf(dst_mac_addr_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", + &(g_kni_handle->dst_mac_addr[0]), &(g_kni_handle->dst_mac_addr[1]), + &(g_kni_handle->dst_mac_addr[2]), &(g_kni_handle->dst_mac_addr[3]), + &(g_kni_handle->dst_mac_addr[4]), &(g_kni_handle->dst_mac_addr[5])); + if(ret != 6){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: dst_mac_addr = invalid, ret = %d, profile = %s, section = %s", ret, profile, section); + goto error_out; + } //init http_project id = http_project_init(); if(id < 0){ @@ -2052,13 +2055,29 @@ extern "C" int kni_init(){ } //init marsio - g_kni_handle->marsio_handle = kni_marsio_init(profile, tfe_node_count); - if(g_kni_handle->marsio_handle == NULL){ - KNI_LOG_ERROR(local_logger, "Failed at init marsio"); - goto error_out; + if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_NORMAL){ + g_kni_handle->marsio_handle = kni_marsio_init(profile, tfe_node_count); + if(g_kni_handle->marsio_handle == NULL){ + KNI_LOG_ERROR(local_logger, "Failed at init marsio"); + goto error_out; + } } - g_kni_handle->tap_handle = kni_tap_init() + //init tun + if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_TUN){ + char tun_name[KNI_SYMBOL_MAX]; + ret = MESA_load_profile_string_nodef(profile, section, "tun_name", tun_name, sizeof(tun_name)); + if(ret < 0){ + KNI_LOG_ERROR(local_logger, "MESA_prof_load: tun_name not set, profile = %s, section = %s", profile, section); + goto error_out; + } + KNI_LOG_ERROR(local_logger, "MESA_prof_load, [%s]:\n tun_name: %s", section, tun_name); + g_kni_handle->tun_handle = kni_tun_init(tun_name, KNI_TUN_MODE_NOBLOCK, local_logger); + if(g_kni_handle->tun_handle == NULL){ + KNI_LOG_ERROR(local_logger, "Failed at init kni_tun"); + goto error_out; + } + } //init maat g_kni_handle->maat_handle = kni_maat_init(profile, local_logger, g_kni_handle->thread_count); @@ -2118,7 +2137,7 @@ extern "C" int kni_init(){ } //init tfe_mgr - _tfe_mgr = tfe_mgr_init(tfe_node_count, profile, local_logger); + _tfe_mgr = tfe_mgr_init(tfe_node_count, profile, g_kni_handle->deploy_mode, local_logger); if(_tfe_mgr == NULL){ KNI_LOG_ERROR(local_logger, "Failed at init tfe_mgr"); goto error_out; diff --git a/entry/src/kni_tap.cpp b/entry/src/kni_tap.cpp deleted file mode 100644 index d80aa28..0000000 --- a/entry/src/kni_tap.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include -#include -#include -#include -#include "kni_utils.h" -#include "MESA_prof_load.h" - -struct kni_tap_handle{ - int fd; - void *logger; -}; - -struct kni_tap_handle* kni_tap_init(void *logger){ - struct kni_tap_handle * tap_handle = (struct kni_tap_handle*)malloc(sizeof(struct kni_tap_handle)); - - char tap_path[1024] = "tap"; - char tap_name[IFNAMSIZ] = {0}; - struct ifreq ifr; - int err = 0; - MESA_load_profile_string_def(".kniconf/kni.conf", "tap", (char*)"tap_path", tap_path, 1024, "/dev/net/tap"); - MESA_load_profile_string_def(".kniconf/kni.conf", "tap", (char*)"tap_name", tap_name, 1024, "/dev/net/tap"); - memset(&ifr, 0, sizeof(ifr)); - ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_ONE_QUEUE; - if(*tap_name){ - strncpy(ifr.ifr_name, tap_name, IFNAMSIZ); - } - if((tap_handle ->fd = open(tap_path, O_RDWR)) < 0){ - KNI_LOG_ERROR(logger, "kni_tap_init():open error,errno is:%d,%s",errno,strerror(errno)); - free(tap_handle); - return NULL; - } - err = ioctl(tap_handle ->fd, TUNSETIFF, (void *)&ifr); - if(err) { - KNI_LOG_ERROR(logger ,"kni_tap_init():ioctl error,errno is:%d,%s",errno,strerror(errno)); - close(tap_handle ->fd); - free(tap_handle); - return NULL; - } - tap_handle->logger = logger; - retrun tap_handle; -} - - -int kni_tap_write(struct kni_tap_handle *handle, char *buff, uint16_t buff_len){ - uint16_t send_len = write(handle->fd, buff, buff_len); - if(send_len < 0){ - KNI_LOG_ERROR(handle->logger, "Failed at kni_tap_write, errno = %d(%s)", errno, strerror(errno)); - return -1; - } - if(send_len < buff_len){ - KNI_LOG_ERROR(handle->logger, "kni_tap_write: need send %dB, only send %dB", buff_len, send_len); - return -2; - } - return 0; -} - -/* -* > 0 : read data length -* = 0 : read null -* = -1 : error -*/ -int kni_tap_read(struct kni_tap_handle *handle, char *buff, uint16_t buff_len){ -<<<<<<< HEAD - int recv_len = 0; - recv_len = read(handle -> fd, buff, buff_len); - if(recv_len < 0){ -======= - int recv_len=0; - recv_len = read(handle -> fd, buff, buff_len); - if(recv_len <0){ ->>>>>>> feature-tap-mode - KNI_LOG_ERROR(handle -> logger, "kni_tap_read() error %d, %s", errno, strerror(errno)); - return -1; - } - else{ - return recv_len; - } - return 0; -} \ No newline at end of file diff --git a/entry/src/kni_tun.cpp b/entry/src/kni_tun.cpp new file mode 100644 index 0000000..ac6a917 --- /dev/null +++ b/entry/src/kni_tun.cpp @@ -0,0 +1,91 @@ +#include "kni_utils.h" +#include +#include +#include +#include +#include + +struct kni_tun_handle{ + int fd; + void *logger; + enum kni_tun_mode mode; +}; + +void kni_tun_destroy(struct kni_tun_handle *tun_handle){ + if(tun_handle != NULL){ + close(tun_handle->fd); + FREE(&tun_handle); + } +} + +struct kni_tun_handle* kni_tun_init(char *tun_name, enum kni_tun_mode mode, void *logger){ + struct kni_tun_handle *tun_handle = ALLOC(struct kni_tun_handle, 1); + tun_handle->logger = logger; + char *tun_path = (char*)"/dev/net/tun"; + struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); + int fd = -1, ret; + ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_ONE_QUEUE; + if(tun_name == NULL || *tun_name == '\0'){ + KNI_LOG_ERROR(logger, "Tap device name is NULL"); + goto error_out; + } + strncpy(ifr.ifr_name, tun_name, IFNAMSIZ); + fd = open(tun_path, O_RDWR); + if(fd < 0){ + KNI_LOG_ERROR(logger, "Failed at open file, filename = %s, errno = %d(%s)", tun_path, errno, strerror(errno)); + goto error_out; + } + tun_handle->fd = fd; + tun_handle->mode = KNI_TUN_MODE_NOBLOCK; + if(tun_handle->mode == KNI_TUN_MODE_NOBLOCK){ + int flags = fcntl(fd, F_GETFL, 0); + ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK); + if(ret < 0){ + KNI_LOG_ERROR(logger ,"Failed at fcntl, fd = %d, flags = F_GETFL | O_NONBLOCK, errno = %d(%s)", errno, strerror(errno)); + goto error_out; + } + } + ret = ioctl(tun_handle->fd, TUNSETIFF, (void *)&ifr); + if(ret < 0){ + KNI_LOG_ERROR(logger ,"Failed at ioctl, fd = %d, errno = %d(%s)", errno, strerror(errno)); + goto error_out; + } + return tun_handle; + +error_out: + kni_tun_destroy(tun_handle); + return NULL; +} + + +int kni_tun_write(struct kni_tun_handle *handle, char *buff, uint16_t buff_len){ + uint16_t ret = write(handle->fd, buff, buff_len); + if(ret < 0){ + KNI_LOG_ERROR(handle->logger, "Failed at write, errno = %d(%s)", errno, strerror(errno)); + return -1; + } + if(ret < buff_len){ + KNI_LOG_ERROR(handle->logger, "Failed at write: need send %dB, only send %dB", buff_len, ret); + return -1; + } + return 0; +} + +/* +* >= 0 : read data length +* = -1 : error +*/ +int kni_tun_read(struct kni_tun_handle *handle, char *buff, uint16_t buff_len){ + int ret = 0; + ret = read(handle->fd, buff, buff_len); + if(ret < 0){ + if(ret == -1 && handle->mode == KNI_TUN_MODE_NOBLOCK){ + if(errno == EWOULDBLOCK || errno == EAGAIN){ + return 0; + } + } + KNI_LOG_ERROR(handle -> logger, "Failed at read, error = %d(%s)", errno, strerror(errno)); + } + return ret; +} \ No newline at end of file diff --git a/entry/src/tap_mode.cpp b/entry/src/tap_mode.cpp deleted file mode 100644 index 51f910b..0000000 --- a/entry/src/tap_mode.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include -#include -#include -#include -#include -#include "kni_utils.h" -#include "MESA_prof_load.h" -struct tap_mode_handle{ - int fd; - void *logger; -}; - -struct tap_mode_handle* tap_mode_init(void *logger){ - struct tap_mode_handle * tap_handle = (struct tap_mode_handle*)malloc(sizeof(struct tap_mode_handle)); - - char tap_path[1024] = {0}; - char tap_name[IFNAMSIZ] = {0}; - struct ifreq ifr; - int err = 0; - - MESA_load_profile_string_def(".kniconf/kni.conf","tap",(char*)"tap_path",tap_path,1024,"/dev/net/tap"); - MESA_load_profile_string_def(".kniconf/kni.conf","tap",(char*)"tap_name",tap_name,1024,"/dev/net/tap"); - - memset(&ifr, 0, sizeof(ifr)); - ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_ONE_QUEUE; - - if(*tap_name) - { - strncpy(ifr.ifr_name, tap_name, IFNAMSIZ); - } - - if((tap_handle ->fd = open(tap_path, O_RDWR)) < 0) - { - KNI_LOG_ERROR(logger, "tap_mode_init():open error,errno is:%d,%s",errno,strerror(errno)); - free(tap_handle); - return NULL; - } - err = ioctl(tap_handle ->fd, TUNSETIFF, (void *)&ifr); - if (err) - { - KNI_LOG_ERROR(logger ,"tap_mode_init():ioctl error,errno is:%d,%s",errno,strerror(errno)); - close(tap_handle ->fd); - free(tap_handle); - return NULL; - } - tap_handle -> logger = logger; - retrun tap_handle; -} - - - -/* -* > 0 : send data length -* = 0 : send null -* = -1 : error -*/ -int tap_mode_write(struct tap_mode_handle *handle, char *buff, uint16_t buff_len){ - int sendlen=0; - - sendlen = write(handle -> fd, buff, buff_len); - if(sendlen < 0) - { - KNI_LOG_ERROR(handle -> logger, "tap_mode_write() error %d, %s",errno,strerror(errno)); - return -1; - } - else - { - return sendlen; - } - - return 0; -} - -/* -* > 0 : read data length -* = 0 : read null -* = -1 : error -*/ -int tap_mode_read(struct tap_mode_handle *handle, char *buff, uint16_t buff_len){ - int recv_len=0; - - recv_len = read(handle -> fd, buff, buff_len); - if(recv_len <0) - { - KNI_LOG_ERROR(handle -> logger, "tap_mode_read() error %d, %s",errno,strerror(errno)); - return -1; - } - else - { - return recv_len; - } - - return 0; -} \ No newline at end of file diff --git a/entry/src/tfe_mgr.cpp b/entry/src/tfe_mgr.cpp index 1cb50c7..3ba3287 100644 --- a/entry/src/tfe_mgr.cpp +++ b/entry/src/tfe_mgr.cpp @@ -284,7 +284,7 @@ error_out: return -1; } -struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logger){ +struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, enum kni_deploy_mode depoly_mode, void *logger){ struct tfe_mgr* mgr = ALLOC(struct tfe_mgr, 1); mgr->logger = logger; int ret; @@ -329,22 +329,27 @@ struct tfe_mgr* tfe_mgr_init(int tfe_node_count, const char* profile, void *logg //load tfe_ipaddr j = 0; for(int i = 0; i < tfe_node_count; i++){ - snprintf(section, sizeof(section), "tfe%d", i); - MESA_load_profile_int_def(profile, section, "enabled", &tfe_node_enabled, 1); - if(tfe_node_enabled != 1){ - continue; + if(depoly_mode == KNI_DEPLOY_MODE_NORMAL){ + snprintf(section, sizeof(section), "tfe%d", i); + MESA_load_profile_int_def(profile, section, "enabled", &tfe_node_enabled, 1); + if(tfe_node_enabled != 1){ + continue; + } + ret = MESA_load_profile_string_nodef(profile, section, "ip_addr", tfe_ipaddr_str, sizeof(tfe_ipaddr_str)); + if(ret < 0){ + KNI_LOG_ERROR(logger, "MESA_prof_load: ip_addr not set, profile is %s, section is %s", profile, section); + goto error_out; + } + KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n ip_addr: %s", section, tfe_ipaddr_str); + ret = inet_pton(AF_INET, tfe_ipaddr_str, &tfe_node_ipaddr); + if(ret != 1){ + KNI_LOG_ERROR(logger, "Failed at inet_pton, ret is %d, errno is %d, errmsg is %s, tfe_id is %d, ip_addr is %s", + ret, errno, strerror(errno), i, tfe_ipaddr_str); + goto error_out; + } } - ret = MESA_load_profile_string_nodef(profile, section, "ip_addr", tfe_ipaddr_str, sizeof(tfe_ipaddr_str)); - if(ret < 0){ - KNI_LOG_ERROR(logger, "MESA_prof_load: ip_addr not set, profile is %s, section is %s", profile, section); - goto error_out; - } - KNI_LOG_ERROR(logger, "MESA_prof_load, [%s]:\n ip_addr: %s", section, tfe_ipaddr_str); - ret = inet_pton(AF_INET, tfe_ipaddr_str, &tfe_node_ipaddr); - if(ret != 1){ - KNI_LOG_ERROR(logger, "Failed at inet_pton, ret is %d, errno is %d, errmsg is %s, tfe_id is %d, ip_addr is %s", - ret, errno, strerror(errno), i, tfe_ipaddr_str); - goto error_out; + else{ + tfe_node_ipaddr = keepalive_listen_ip; } mgr->tfe_enabled_nodes[j].tfe_id = i; mgr->tfe_enabled_nodes[j].ipaddr = tfe_node_ipaddr;