#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "log.h" #include "uthash.h" #include "bfd.h" #include "sf_status.h" #include "utils.h" #include "health_check.h" #define BUF_SIZE 4096 #define SEND_MAX 1 #define PACKET_SIZE 64 #define HC_DEV_NAME_LEN 16 #define HC_LOCAL_ADDRESS_LEN 64 #define TIMESPEC_TO_MSEC(ts) ((ts).tv_sec * 1000 + (ts).tv_nsec / 1000000) struct session_table { // handler; struct session_iterm *root_by_id; pthread_rwlock_t rwlock; }; struct session_iterm { uint64_t session_id; // key struct health_check policy; // value1: deep copy int is_active; // value2 uuid_t sf_uuid; // value3 int vsys_id; // value4 UT_hash_handle hh1; /* handle for first hash table */ }; struct session_table_addr { // handler; struct node_addr *htable; pthread_rwlock_t rwlock; }; struct node_addr { char address[64]; // key uint8_t mac[ETH_ALEN]; int ref_cnt; // reference UT_hash_handle hh; /* handle for first hash table */ }; static uint64_t g_session_id; static struct session_table g_handle; static struct session_table_addr g_handle_bfd; static struct session_table_addr g_handle_none; static struct sf_status *g_sf_status = NULL; int next_check_wait_ms = 300; int enable = 1; int icmp_cycle_time_s = 10; char path[BFD_PATHLEN]; char hc_dev_name[HC_DEV_NAME_LEN]; char local_address[HC_LOCAL_ADDRESS_LEN]; char gateway_address[HC_LOCAL_ADDRESS_LEN]; uint8_t default_gw_mac[ETH_ALEN]; static int get_mac_by_addr(char *addr, uint8_t *buf); static int health_check_session_foreach(); static int health_check_method_table_set_mac(struct session_table_addr *table, char *addr, uint8_t *mac) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } memcpy(tmp->mac, mac, ETH_ALEN); pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_get_mac(struct session_table_addr *table, char *addr, uint8_t *out_mac) { struct node_addr *tmp = NULL; pthread_rwlock_rdlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } memcpy(out_mac, tmp->mac, ETH_ALEN); pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_del(struct session_table_addr *table, char *addr) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 0; } if (--tmp->ref_cnt) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } HASH_DEL(table->htable, tmp); free(tmp); tmp = NULL; pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_add(struct session_table_addr *table, char *addr) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp) { tmp->ref_cnt++; } else { tmp = (struct node_addr *)calloc(1, sizeof(struct node_addr)); assert(tmp); snprintf(tmp->address, sizeof(tmp->address), addr); tmp->ref_cnt++; HASH_ADD_STR(table->htable, address, tmp); } pthread_rwlock_unlock(&(table->rwlock)); return 0; } #define CHECKSUM_CARRY(x) (x = (x >> 16) + (x & 0xffff), (~(x + (x >> 16)) & 0xffff)) static inline int checksum(uint16_t *data, int len) { int sum = 0; int nleft = len; uint16_t ans = 0; uint16_t *w = data; while (nleft > 1) { sum += *w++; nleft -= 2; } if (nleft == 1) { *(char *)(&ans) = *(char *)w; sum += ans; } return sum; } static int send_icmp_pkt(char *addr) { int sockfd; char packet[PACKET_SIZE] = {0}; struct ifreq ifr; struct icmp *icmp = (struct icmp *)packet; struct sockaddr_in dest_addr; sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); if (sockfd < 0) return 1; dest_addr.sin_family = AF_INET; dest_addr.sin_port = 0; if (inet_pton(AF_INET, addr, &(dest_addr.sin_addr)) <= 0) { close(sockfd); LOG_ERROR("unable to send icmp packet, address[%s] invalid!", addr); return 1; } memset(&ifr, 0, sizeof(ifr)); snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s", hc_dev_name); if (setsockopt(sockfd, SOL_SOCKET, SO_BINDTODEVICE, (void *)&ifr, sizeof(ifr)) < 0) { close(sockfd); LOG_ERROR("unable to send icmp packet, device[%s] bind failed!", hc_dev_name); return 1; } icmp->icmp_type = ICMP_ECHO; icmp->icmp_code = 0; icmp->icmp_id = htons(getpid()); icmp->icmp_seq = 0; icmp->icmp_cksum = 0; int sum = checksum((uint16_t *)icmp, PACKET_SIZE); icmp->icmp_cksum = CHECKSUM_CARRY(sum); for(int i=0; inlmsg_type == NLMSG_DONE) break; if (nh->nlmsg_type == NLMSG_ERROR) { close(nl_sock); goto start; } struct ndmsg *nd = (struct ndmsg *)NLMSG_DATA(nh); struct rtattr *rth = (struct rtattr *)RTM_RTA(nd); int rtl = RTM_PAYLOAD(nh); memset(&ipaddr, 0, sizeof(ipaddr)); mac = NULL; for (; RTA_OK(rth, rtl); rth = RTA_NEXT(rth, rtl)) { if (rth->rta_type == NDA_DST) { memcpy(&ipaddr, RTA_DATA(rth), sizeof(struct in_addr)); } else if (rth->rta_type == NDA_LLADDR) { mac = (uint8_t *)RTA_DATA(rth); } } if (inet_ntop(AF_INET, &ipaddr, str_addr, INET_ADDRSTRLEN) != NULL) { if (mac == NULL) { health_check_method_table_set_mac(&g_handle_bfd, str_addr, init_mac); health_check_method_table_set_mac(&g_handle_none, str_addr, init_mac); } else { health_check_method_table_set_mac(&g_handle_bfd, str_addr, mac); health_check_method_table_set_mac(&g_handle_none, str_addr, mac); } } } } close(nl_sock); return 0; } static int listen_arp_table() { pthread_t pid; pthread_create(&pid, NULL, _listen_arp_table, NULL); pthread_detach(pid); return pid; } static void *_send_icmp_cycle(void *arg) { struct node_addr *node = NULL; struct node_addr *tmp = NULL; while (1) { pthread_rwlock_rdlock(&g_handle_none.rwlock); HASH_ITER(hh, g_handle_none.htable, node, tmp) { send_icmp_pkt(node->address); } pthread_rwlock_unlock(&g_handle_none.rwlock); sleep(icmp_cycle_time_s); } } static int send_icmp_cycle() { pthread_t pid; pthread_create(&pid, NULL, _send_icmp_cycle, NULL); pthread_detach(pid); return pid; } void health_check_session_init(const char *profile, struct kafka *kfk) { char default_gw_mac_str[32] = { 0 }; memset(&g_handle, 0, sizeof(g_handle)); pthread_rwlock_init(&g_handle.rwlock, NULL); memset(&g_handle_bfd, 0, sizeof(g_handle_bfd)); pthread_rwlock_init(&g_handle_bfd.rwlock, NULL); memset(&g_handle_none, 0, sizeof(g_handle_none)); pthread_rwlock_init(&g_handle_none.rwlock, NULL); MESA_load_profile_int_def(profile, "bfdd", "enable", &enable, 1); MESA_load_profile_int_def(profile, "bfdd", "icmp_cycle_time_s", &icmp_cycle_time_s, 10); MESA_load_profile_string_def(profile, "bfdd", "path", path, sizeof(path), "/var/run/frr/bfdd.vty"); MESA_load_profile_string_def(profile, "bfdd", "device", hc_dev_name, sizeof(hc_dev_name), "eth0"); MESA_load_profile_string_def(profile, "bfdd", "local_address", local_address, sizeof(local_address), "127.0.0.1"); MESA_load_profile_string_nodef(profile, "bfdd", "gateway", gateway_address, sizeof(gateway_address)); MESA_load_profile_string_def(profile, "bfdd", "default_gw_mac", default_gw_mac_str, sizeof(default_gw_mac_str), "aa:aa:aa:aa:aa:aa"); if (enable == 0) { str_to_mac(default_gw_mac_str, default_gw_mac); return; } g_sf_status = sf_status_create(profile, kfk); if (strlen(gateway_address) > 0) { health_check_method_table_add(&g_handle_none, gateway_address); } health_check_session_foreach(); listen_arp_table(); send_icmp_cycle(); } static int health_check_session_recover_cfg(struct bfd_vtysh_client *client) { int ret = 0; struct session_iterm *tmp = NULL; struct session_iterm *node = NULL; HASH_ITER(hh1, g_handle.root_by_id, node, tmp) { if (node->policy.method != HEALTH_CHECK_METHOD_BFD) continue; ret = bfd_vtysh_add_dev(client, node->policy.address, node->policy.retires, node->policy.interval_ms); if (ret != 0) return -1; } return 0; } static void health_check_session_init_bfd_client(struct bfd_vtysh_client *client) { memset(client, 0, sizeof(*client)); snprintf(client->path, sizeof(client->path), path); client->pre_config = bfd_vtysh_pre_config; } static void health_check_session_recover_bfd(struct bfd_vtysh_client *client) { memset(client, 0, sizeof(*client)); snprintf(client->path, sizeof(client->path), path); client->pre_config = bfd_vtysh_pre_config; client->recover_config = health_check_session_recover_cfg; } static int bfd_rule_add(const struct health_check *policy) { int ret = 0; struct bfd_vtysh_client client; health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); ret = bfd_vtysh_add_dev(&client, policy->address, policy->retires, policy->interval_ms); if (ret != 0) LOG_ERROR("bfd vtysh add dev address [%s] failed!", policy->address); bfd_vtysh_close(&client); return ret; } static int bfd_rule_del(const struct health_check *policy) { int ret = 0; struct bfd_vtysh_client client; health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); ret = bfd_vtysh_del_dev(&client, policy->address); if (ret != 0) { LOG_ERROR("bfd vtysh delete dev address [%s] failed!", policy->address); } bfd_vtysh_close(&client); return ret; } static struct session_iterm *health_check_session_get(uint64_t session_id) { struct session_iterm *tmp = NULL; HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); return tmp; } static uint64_t health_check_get_session_id() { struct session_iterm *tmp = NULL; uint64_t tmp_session_id = g_session_id; while(1) { g_session_id++; if (g_session_id == 0) g_session_id++; if (tmp_session_id == g_session_id) return 0; tmp = health_check_session_get(g_session_id); if (tmp) continue; break; } return g_session_id; } // return >0 : session id // return 0 : fail // struct health_check *policy : need deep copy uint64_t health_check_session_add(uuid_t *sf_uuid, int vsys_id, const struct health_check *policy) { uint64_t session_id = 0; uint8_t mac[ETH_ALEN] = {0}; struct session_iterm *tmp = NULL; if (enable == 0) { return 1; } pthread_rwlock_wrlock(&g_handle.rwlock); session_id = health_check_get_session_id(); if (session_id == 0) { pthread_rwlock_unlock(&g_handle.rwlock); LOG_ERROR("health check get session id failed!"); return 0; } tmp = (struct session_iterm *)calloc(1, sizeof(struct session_iterm)); assert(tmp); tmp->vsys_id = vsys_id; tmp->session_id = session_id; uuid_copy(tmp->sf_uuid, *sf_uuid); memcpy(&tmp->policy, policy, sizeof(struct health_check)); HASH_ADD(hh1, g_handle.root_by_id, session_id, sizeof(tmp->session_id), tmp); pthread_rwlock_unlock(&g_handle.rwlock); if (policy->method == HEALTH_CHECK_METHOD_BFD) { health_check_method_table_add(&g_handle_bfd, tmp->policy.address); bfd_rule_add(policy); } else if (policy->method == HEALTH_CHECK_METHOD_NONE) { health_check_method_table_add(&g_handle_none, tmp->policy.address); send_icmp_pkt(tmp->policy.address); get_mac_by_addr(tmp->policy.address, mac); health_check_method_table_set_mac(&g_handle_none, tmp->policy.address, mac); } char sf_uuid_str[UUID_STRING_SIZE] = {0}; uuid_unparse(*sf_uuid, sf_uuid_str); LOG_DEBUG("health check session table insert: profile id [%s] session id [%lu] address [%s] success", sf_uuid_str, session_id, policy->address); return session_id; } // return 0 : success // return -1 : key not exist int health_check_session_del(uint64_t session_id, uuid_t *sf_uuid, int vsys_id) { int ret = 0; struct session_iterm *tmp = NULL; if (enable == 0) { return 0; } pthread_rwlock_wrlock(&g_handle.rwlock); tmp = health_check_session_get(session_id); if (!tmp) { pthread_rwlock_unlock(&g_handle.rwlock); LOG_DEBUG("health check session table delete: session id [%lu] not exists", session_id); return -1; } if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD) { ret = health_check_method_table_del(&g_handle_bfd, tmp->policy.address); if (ret != 0) goto end; bfd_rule_del(&tmp->policy); } else if (tmp->policy.method == HEALTH_CHECK_METHOD_NONE) { ret = health_check_method_table_del(&g_handle_none, tmp->policy.address); } end: HASH_DELETE(hh1, g_handle.root_by_id, tmp); struct sf_status_key key = {0}; key.vsys_id = vsys_id; uuid_copy(key.sf_uuid, *sf_uuid); sf_status_delete(g_sf_status, &key); pthread_rwlock_unlock(&g_handle.rwlock); free(tmp); tmp = NULL; char sf_uuid_str[UUID_STRING_SIZE] = {0}; uuid_unparse(*sf_uuid, sf_uuid_str); LOG_DEBUG("health check session table delete: profile id [%s] session id [%lu] success", sf_uuid_str, session_id); return 0; } // return 1 : active // return 0 : inactive // return -1 : key not exist int health_check_session_get_status(uint64_t session_id) { int status = 0; struct session_iterm *tmp = NULL; if (enable == 0) { return 1; } pthread_rwlock_rdlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session table get status: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } status = tmp->is_active; pthread_rwlock_unlock(&g_handle.rwlock); LOG_DEBUG("health check session id[%lu] get status [%d]", session_id, status); return status; } // return 0 : success // return -1 : key not exist int health_check_session_set_status(uint64_t session_id, int is_active) { struct session_iterm *tmp = NULL; if (enable == 0) { return 0; } pthread_rwlock_wrlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session table set status: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } tmp->is_active = is_active; pthread_rwlock_unlock(&g_handle.rwlock); return 0; } static int get_mac_by_addr(char *addr, uint8_t *buf) { int sfd, ret; struct arpreq arp_req; struct sockaddr_in *sin; sin = (struct sockaddr_in *)&(arp_req.arp_pa); memset(&arp_req, 0, sizeof(arp_req)); sin->sin_family = AF_INET; inet_pton(AF_INET, addr, &(sin->sin_addr)); snprintf(arp_req.arp_dev, IFNAMSIZ, hc_dev_name); sfd = socket(AF_INET, SOCK_DGRAM, 0); if (sfd == -1) return -1; ret = ioctl(sfd, SIOCGARP, &arp_req); if (ret == 0) memcpy(buf, arp_req.arp_ha.sa_data, ETH_ALEN); LOG_DEBUG("IP:%s, MAC: %02x:%02x:%02x:%02x:%02x:%02x", addr, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]); close(sfd); return 0; } static void *_health_check_session_foreach(void *arg) { int is_active = 0; int ouput_interval_ms = sf_status_get_ouput_interval_ms(g_sf_status); struct bfd_vtysh_client client; struct session_iterm *tmp = NULL; struct session_iterm *node = NULL; uint8_t mac[ETH_ALEN] = {0}; uint8_t init_mac[ETH_ALEN] = {0}; struct sockaddr_in addr; struct timespec current_time; struct timespec last_output_time; clock_gettime(CLOCK_MONOTONIC, ¤t_time); last_output_time = current_time; health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); while(1) { pthread_rwlock_wrlock(&g_handle.rwlock); HASH_ITER(hh1, g_handle.root_by_id, node, tmp) { if (node->policy.method != HEALTH_CHECK_METHOD_BFD) continue; if (inet_pton(AF_INET, node->policy.address, &(addr.sin_addr)) > 0) { is_active = bfd_vtysh_get_dev_active(&client, node->policy.address); if (is_active == -1) { bfd_vtysh_close(&client); health_check_session_recover_bfd(&client); bfd_vtysh_connect(&client); is_active = bfd_vtysh_get_dev_active(&client, node->policy.address); if (is_active == -1) is_active = 0; } } else { is_active = 0; } struct sf_status_key key = {0}; key.vsys_id = node->vsys_id; uuid_copy(key.sf_uuid, node->sf_uuid); sf_status_update(g_sf_status, &key, is_active, 0); if (node->is_active != is_active) { node->is_active = is_active; if (node->is_active == 1) { memset(mac, 0, ETH_ALEN); get_mac_by_addr(node->policy.address, mac); health_check_method_table_set_mac(&g_handle_bfd, node->policy.address, mac); } else { health_check_method_table_set_mac(&g_handle_bfd, node->policy.address, init_mac); } } if (next_check_wait_ms > node->policy.interval_ms) next_check_wait_ms = node->policy.interval_ms; } pthread_rwlock_unlock(&g_handle.rwlock); clock_gettime(CLOCK_MONOTONIC, ¤t_time); int next_output_wait_ms = ouput_interval_ms - (TIMESPEC_TO_MSEC(current_time) - TIMESPEC_TO_MSEC(last_output_time)); if (next_output_wait_ms <= 0) { next_output_wait_ms = 0; } if (next_output_wait_ms >= next_check_wait_ms) { usleep(next_check_wait_ms * 1000); } else { usleep(next_output_wait_ms * 1000); clock_gettime(CLOCK_MONOTONIC, ¤t_time); sf_status_output(g_sf_status); last_output_time = current_time; usleep((next_check_wait_ms - next_output_wait_ms) * 1000); } } bfd_vtysh_close(&client); return NULL; } static int health_check_session_foreach() { pthread_t pid; pthread_create(&pid, NULL, _health_check_session_foreach, NULL); pthread_detach(pid); return pid; } static const char *health_check_method_str(enum health_check_method method) { switch (method) { case HEALTH_CHECK_METHOD_NONE: return "HEALTH_CHECK_METHOD_NONE"; case HEALTH_CHECK_METHOD_IN_BAND_BFD: return "HEALTH_CHECK_METHOD_IN_BAND_BFD"; case HEALTH_CHECK_METHOD_BFD: return "HEALTH_CHECK_METHOD_BFD"; case HEALTH_CHECK_METHOD_HTTP: return "HEALTH_CHECK_METHOD_HTTP"; default: return NULL; } } // return 0 : success // return -1 : key not exist int health_check_session_get_mac(uint64_t session_id, u_char mac_buff[]) { const char *str_method = NULL; struct session_iterm *tmp = NULL; uint8_t mac[ETH_ALEN] = {0}; uint8_t init_mac[ETH_ALEN] = {0}; char sf_uuid_str[UUID_STRING_SIZE] = {0}; if (enable == 0) { memcpy(mac_buff, default_gw_mac, ETH_ALEN); return 0; } pthread_rwlock_rdlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session get mac: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } uuid_unparse(tmp->sf_uuid, sf_uuid_str); str_method = health_check_method_str(tmp->policy.method); if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD && tmp->is_active == 0) { LOG_DEBUG("health check session id [%lu] profile id [%s] health check method [%s] active is down", session_id, sf_uuid_str, str_method); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD) { health_check_method_table_get_mac(&g_handle_bfd, tmp->policy.address, mac); } else if (tmp->policy.method == HEALTH_CHECK_METHOD_NONE) { health_check_method_table_get_mac(&g_handle_none, tmp->policy.address, mac); } if (memcmp(mac, init_mac, ETH_ALEN) == 0) { if (strlen(gateway_address) == 0) { LOG_DEBUG("health check session id [%lu] profile id [%s] health check method [%s] get mac [null]", session_id, sf_uuid_str, str_method); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } health_check_method_table_get_mac(&g_handle_none, gateway_address, mac); if (memcmp(mac, init_mac, ETH_ALEN) == 0) { LOG_DEBUG("health check session id [%lu] profile id [%s] health check method [%s] get mac [null]", session_id, sf_uuid_str, str_method); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } } memcpy(mac_buff, mac, ETH_ALEN); LOG_DEBUG("health check session id [%lu] profile id [%s] health check method [%s] get mac [%02x:%02x:%02x:%02x:%02x:%02x]", session_id, sf_uuid_str, str_method, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); pthread_rwlock_unlock(&g_handle.rwlock); return 0; }