#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "log.h" #include "uthash.h" #include "bfd.h" #include "sf_status.h" #include "utils.h" #include "health_check.h" #define BUF_SIZE 4096 #define SEND_MAX 1 #define PACKET_SIZE 64 #define HC_MAC_LEN 6 #define HC_DEV_NAME_LEN 16 #define HC_LOCAL_ADDRESS_LEN 64 struct session_table { // handler; struct session_iterm *root_by_id; pthread_rwlock_t rwlock; }; struct session_iterm { uint64_t session_id; // key struct health_check policy; // value1: deep copy int is_active; // value2 int profile_id; // value3 int vsys_id; // value4 UT_hash_handle hh1; /* handle for first hash table */ }; struct session_table_addr { // handler; struct node_addr *htable; pthread_rwlock_t rwlock; }; struct node_addr { char address[64]; // key uint8_t mac[HC_MAC_LEN]; int ref_cnt; // reference UT_hash_handle hh; /* handle for first hash table */ }; static uint64_t g_session_id; static struct session_table g_handle; static struct session_table_addr g_handle_bfd; static struct session_table_addr g_handle_none; static struct sf_status *g_sf_status = NULL; int sleep_ms = 300; int enable = 1; int icmp_cycle_time_s = 10; char path[BFD_PATHLEN]; char hc_dev_name[HC_DEV_NAME_LEN]; char local_address[HC_LOCAL_ADDRESS_LEN]; char gateway_address[HC_LOCAL_ADDRESS_LEN]; uint8_t default_gw_mac[HC_MAC_LEN]; static int get_mac_by_addr(char *addr, uint8_t *buf); static int health_check_session_foreach(); static int health_check_method_table_set_mac(struct session_table_addr *table, char *addr, uint8_t *mac) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } memcpy(tmp->mac, mac, HC_MAC_LEN); pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_get_mac(struct session_table_addr *table, char *addr, uint8_t *out_mac) { struct node_addr *tmp = NULL; pthread_rwlock_rdlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } memcpy(out_mac, tmp->mac, HC_MAC_LEN); pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_del(struct session_table_addr *table, char *addr) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp == NULL) { pthread_rwlock_unlock(&(table->rwlock)); return 0; } if (--tmp->ref_cnt) { pthread_rwlock_unlock(&(table->rwlock)); return 1; } HASH_DEL(table->htable, tmp); free(tmp); tmp = NULL; pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int health_check_method_table_add(struct session_table_addr *table, char *addr) { struct node_addr *tmp = NULL; pthread_rwlock_wrlock(&(table->rwlock)); HASH_FIND_STR(table->htable, addr, tmp); if (tmp) { tmp->ref_cnt++; } else { tmp = (struct node_addr *)calloc(1, sizeof(struct node_addr)); assert(tmp); snprintf(tmp->address, sizeof(tmp->address), addr); tmp->ref_cnt++; HASH_ADD_STR(table->htable, address, tmp); } pthread_rwlock_unlock(&(table->rwlock)); return 0; } static int send_icmp_pkt(char *addr) { int sockfd; char packet[PACKET_SIZE] = {0}; struct ifreq ifr; struct icmp *icmp = (struct icmp *)packet; struct sockaddr_in dest_addr; sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP); if (sockfd < 0) return 1; dest_addr.sin_family = AF_INET; dest_addr.sin_port = 0; if (inet_pton(AF_INET, addr, &(dest_addr.sin_addr)) <= 0) { close(sockfd); LOG_ERROR("unable to send icmp packet, address[%s] invalid!", addr); return 1; } memset(&ifr, 0, sizeof(ifr)); snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s", hc_dev_name); if (setsockopt(sockfd, SOL_SOCKET, SO_BINDTODEVICE, (void *)&ifr, sizeof(ifr)) < 0) { close(sockfd); LOG_ERROR("unable to send icmp packet, device[%s] bind failed!", hc_dev_name); return 1; } icmp->icmp_type = ICMP_ECHO; icmp->icmp_code = 0; icmp->icmp_id = htons(getpid()); icmp->icmp_seq = 0; icmp->icmp_cksum = 0; int sum = checksum((uint16_t *)icmp, PACKET_SIZE); icmp->icmp_cksum = CHECKSUM_CARRY(sum); for(int i=0; inlmsg_type == NLMSG_DONE) break; if (nh->nlmsg_type == NLMSG_ERROR) { close(nl_sock); goto start; } struct ndmsg *nd = (struct ndmsg *)NLMSG_DATA(nh); struct rtattr *rth = (struct rtattr *)RTM_RTA(nd); int rtl = RTM_PAYLOAD(nh); memset(&ipaddr, 0, sizeof(ipaddr)); mac = NULL; for (; RTA_OK(rth, rtl); rth = RTA_NEXT(rth, rtl)) { if (rth->rta_type == NDA_DST) { memcpy(&ipaddr, RTA_DATA(rth), sizeof(struct in_addr)); } else if (rth->rta_type == NDA_LLADDR) { mac = (uint8_t *)RTA_DATA(rth); } } if (inet_ntop(AF_INET, &ipaddr, str_addr, INET_ADDRSTRLEN) != NULL) { if (mac == NULL) { health_check_method_table_set_mac(&g_handle_bfd, str_addr, init_mac); health_check_method_table_set_mac(&g_handle_none, str_addr, init_mac); } else { health_check_method_table_set_mac(&g_handle_bfd, str_addr, mac); health_check_method_table_set_mac(&g_handle_none, str_addr, mac); } } } } close(nl_sock); return 0; } static int listen_arp_table() { pthread_t pid; pthread_create(&pid, NULL, _listen_arp_table, NULL); pthread_detach(pid); return pid; } static void *_send_icmp_cycle(void *arg) { struct node_addr *node = NULL; struct node_addr *tmp = NULL; while (1) { pthread_rwlock_rdlock(&g_handle_none.rwlock); HASH_ITER(hh, g_handle_none.htable, node, tmp) { send_icmp_pkt(node->address); } pthread_rwlock_unlock(&g_handle_none.rwlock); sleep(icmp_cycle_time_s); } } static int send_icmp_cycle() { pthread_t pid; pthread_create(&pid, NULL, _send_icmp_cycle, NULL); pthread_detach(pid); return pid; } void health_check_session_init(const char *profile) { memset(&g_handle, 0, sizeof(g_handle)); pthread_rwlock_init(&g_handle.rwlock, NULL); memset(&g_handle_bfd, 0, sizeof(g_handle_bfd)); pthread_rwlock_init(&g_handle_bfd.rwlock, NULL); memset(&g_handle_none, 0, sizeof(g_handle_none)); pthread_rwlock_init(&g_handle_none.rwlock, NULL); MESA_load_profile_int_def(profile, "bfdd", "enable", &enable, 1); MESA_load_profile_int_def(profile, "bfdd", "icmp_cycle_time_s", &icmp_cycle_time_s, 10); MESA_load_profile_string_def(profile, "bfdd", "path", path, sizeof(path), "/var/run/frr/bfdd.vty"); MESA_load_profile_string_def(profile, "bfdd", "device", hc_dev_name, sizeof(hc_dev_name), "eth0"); MESA_load_profile_string_def(profile, "bfdd", "local_address", local_address, sizeof(local_address), "127.0.0.1"); MESA_load_profile_string_def(profile, "bfdd", "gateway", gateway_address, sizeof(gateway_address), "127.0.0.1"); if (enable == 0) { return; } g_sf_status = sf_status_create(profile); health_check_method_table_add(&g_handle_none, gateway_address); health_check_session_foreach(); listen_arp_table(); send_icmp_cycle(); } static int health_check_session_recover_cfg(struct bfd_vtysh_client *client) { int ret = 0; struct session_iterm *tmp = NULL; struct session_iterm *node = NULL; HASH_ITER(hh1, g_handle.root_by_id, node, tmp) { if (node->policy.method != HEALTH_CHECK_METHOD_BFD) continue; ret = bfd_vtysh_add_dev(client, node->policy.address, node->policy.retires, node->policy.interval_ms); if (ret != 0) return -1; } return 0; } static void health_check_session_init_bfd_client(struct bfd_vtysh_client *client) { memset(client, 0, sizeof(*client)); snprintf(client->path, sizeof(client->path), path); client->pre_config = bfd_vtysh_pre_config; } static void health_check_session_recover_bfd(struct bfd_vtysh_client *client) { memset(client, 0, sizeof(*client)); snprintf(client->path, sizeof(client->path), path); client->pre_config = bfd_vtysh_pre_config; client->recover_config = health_check_session_recover_cfg; } static int bfd_rule_add(const struct health_check *policy) { int ret = 0; struct bfd_vtysh_client client; health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); ret = bfd_vtysh_add_dev(&client, policy->address, policy->retires, policy->interval_ms); if (ret != 0) LOG_ERROR("bfd vtysh add dev address [%s] failed!", policy->address); bfd_vtysh_close(&client); return ret; } static int bfd_rule_del(const struct health_check *policy) { int ret = 0; struct bfd_vtysh_client client; health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); ret = bfd_vtysh_del_dev(&client, policy->address); if (ret != 0) { LOG_ERROR("bfd vtysh delete dev address [%s] failed!", policy->address); } bfd_vtysh_close(&client); return ret; } static struct session_iterm *health_check_session_get(uint64_t session_id) { struct session_iterm *tmp = NULL; HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); return tmp; } static uint64_t health_check_get_session_id() { struct session_iterm *tmp = NULL; uint64_t tmp_session_id = g_session_id; while(1) { g_session_id++; if (g_session_id == 0) g_session_id++; if (tmp_session_id == g_session_id) return 0; tmp = health_check_session_get(g_session_id); if (tmp) continue; break; } return g_session_id; } // return >0 : session id // return 0 : fail // struct health_check *policy : need deep copy uint64_t health_check_session_add(int profile_id, int vsys_id, const struct health_check *policy) { uint64_t session_id = 0; uint8_t mac[HC_MAC_LEN] = {0}; struct session_iterm *tmp = NULL; if (enable == 0) { return 1; } pthread_rwlock_wrlock(&g_handle.rwlock); session_id = health_check_get_session_id(); if (session_id == 0) { pthread_rwlock_unlock(&g_handle.rwlock); LOG_ERROR("health check get session id failed!"); return 0; } tmp = (struct session_iterm *)calloc(1, sizeof(struct session_iterm)); assert(tmp); tmp->vsys_id = vsys_id; tmp->session_id = session_id; tmp->profile_id = profile_id; memcpy(&tmp->policy, policy, sizeof(struct health_check)); HASH_ADD(hh1, g_handle.root_by_id, session_id, sizeof(tmp->session_id), tmp); pthread_rwlock_unlock(&g_handle.rwlock); if (policy->method == HEALTH_CHECK_METHOD_BFD) { health_check_method_table_add(&g_handle_bfd, tmp->policy.address); bfd_rule_add(policy); } else if (policy->method == HEALTH_CHECK_METHOD_NONE) { health_check_method_table_add(&g_handle_none, tmp->policy.address); send_icmp_pkt(tmp->policy.address); get_mac_by_addr(tmp->policy.address, mac); health_check_method_table_set_mac(&g_handle_none, tmp->policy.address, mac); } LOG_DEBUG("health check session table insert: profile id [%d] session id [%lu] address [%s] success", profile_id, session_id, policy->address); return session_id; } // return 0 : success // return -1 : key not exist int health_check_session_del(uint64_t session_id, int profile_id) { int ret = 0; struct session_iterm *tmp = NULL; if (enable == 0) { return 0; } pthread_rwlock_wrlock(&g_handle.rwlock); tmp = health_check_session_get(session_id); if (!tmp) { pthread_rwlock_unlock(&g_handle.rwlock); LOG_DEBUG("health check session table delete: session id [%lu] not exists", session_id); return -1; } if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD) { ret = health_check_method_table_del(&g_handle_bfd, tmp->policy.address); if (ret != 0) goto end; bfd_rule_del(&tmp->policy); } else if (tmp->policy.method == HEALTH_CHECK_METHOD_NONE) { ret = health_check_method_table_del(&g_handle_none, tmp->policy.address); } end: HASH_DELETE(hh1, g_handle.root_by_id, tmp); sf_status_delete(g_sf_status, profile_id); pthread_rwlock_unlock(&g_handle.rwlock); free(tmp); tmp = NULL; LOG_DEBUG("health check session table delete: profile id [%d] session id [%lu] success", profile_id, session_id); return 0; } // return 1 : active // return 0 : inactive // return -1 : key not exist int health_check_session_get_status(uint64_t session_id) { int status = 0; struct session_iterm *tmp = NULL; if (enable == 0) { return 1; } pthread_rwlock_rdlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session table get status: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } status = tmp->is_active; pthread_rwlock_unlock(&g_handle.rwlock); LOG_DEBUG("health check session id[%lu] get status [%d]", session_id, status); return status; } // return 0 : success // return -1 : key not exist int health_check_session_set_status(uint64_t session_id, int is_active) { struct session_iterm *tmp = NULL; if (enable == 0) { return 0; } pthread_rwlock_wrlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session table set status: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } tmp->is_active = is_active; pthread_rwlock_unlock(&g_handle.rwlock); return 0; } static int get_mac_by_addr(char *addr, uint8_t *buf) { int sfd, ret; struct arpreq arp_req; struct sockaddr_in *sin; sin = (struct sockaddr_in *)&(arp_req.arp_pa); memset(&arp_req, 0, sizeof(arp_req)); sin->sin_family = AF_INET; inet_pton(AF_INET, addr, &(sin->sin_addr)); snprintf(arp_req.arp_dev, IFNAMSIZ, hc_dev_name); sfd = socket(AF_INET, SOCK_DGRAM, 0); if (sfd == -1) return -1; ret = ioctl(sfd, SIOCGARP, &arp_req); if (ret == 0) memcpy(buf, arp_req.arp_ha.sa_data, HC_MAC_LEN); LOG_DEBUG("IP:%s, MAC: %02x:%02x:%02x:%02x:%02x:%02x", addr, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]); close(sfd); return 0; } static void *_health_check_session_foreach(void *arg) { int is_active = 0; int interval_s = sf_status_get_interval(g_sf_status); struct bfd_vtysh_client client; struct session_iterm *tmp = NULL; struct session_iterm *node = NULL; uint8_t mac[HC_MAC_LEN] = {0}; uint8_t init_mac[HC_MAC_LEN] = {0}; struct sockaddr_in addr; struct timespec current_time; struct timespec g_status_last_send_time; clock_gettime(CLOCK_MONOTONIC, ¤t_time); clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time); health_check_session_init_bfd_client(&client); bfd_vtysh_connect(&client); while(1) { pthread_rwlock_wrlock(&g_handle.rwlock); HASH_ITER(hh1, g_handle.root_by_id, node, tmp) { if (node->policy.method != HEALTH_CHECK_METHOD_BFD) continue; if (inet_pton(AF_INET, node->policy.address, &(addr.sin_addr)) > 0) { is_active = bfd_vtysh_get_dev_active(&client, node->policy.address); if (is_active == -1) { bfd_vtysh_close(&client); health_check_session_recover_bfd(&client); bfd_vtysh_connect(&client); is_active = bfd_vtysh_get_dev_active(&client, node->policy.address); if (is_active == -1) is_active = 0; } } else { is_active = 0; } sf_status_update(g_sf_status, node->vsys_id, node->profile_id, is_active, 0); if (node->is_active != is_active) { node->is_active = is_active; if (node->is_active == 1) { memset(mac, 0, HC_MAC_LEN); get_mac_by_addr(node->policy.address, mac); health_check_method_table_set_mac(&g_handle_bfd, node->policy.address, mac); } else { health_check_method_table_set_mac(&g_handle_bfd, node->policy.address, init_mac); } } if (sleep_ms > node->policy.interval_ms) sleep_ms = node->policy.interval_ms; } pthread_rwlock_unlock(&g_handle.rwlock); clock_gettime(CLOCK_MONOTONIC, ¤t_time); if (current_time.tv_sec - g_status_last_send_time.tv_sec >= interval_s) { sf_status_send(g_sf_status); clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time); } // interval_s : 1000 ms // sleep_ms : 900 ms if (interval_s * 1000 > sleep_ms) { usleep(sleep_ms * 1000); } // interval_s : 900 ms // sleep_ms : 1000 ms else { int tmp_time = sleep_ms; while(tmp_time > interval_s * 1000) { usleep(interval_s * 1000 * 1000); clock_gettime(CLOCK_MONOTONIC, ¤t_time); if (current_time.tv_sec - g_status_last_send_time.tv_sec >= interval_s) { sf_status_send(g_sf_status); clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time); } tmp_time -= interval_s * 1000; } usleep(tmp_time * 1000); } } bfd_vtysh_close(&client); return NULL; } static int health_check_session_foreach() { pthread_t pid; pthread_create(&pid, NULL, _health_check_session_foreach, NULL); pthread_detach(pid); return pid; } static const char *health_check_method_str(enum health_check_method method) { switch (method) { case HEALTH_CHECK_METHOD_NONE: return "HEALTH_CHECK_METHOD_NONE"; case HEALTH_CHECK_METHOD_IN_BAND_BFD: return "HEALTH_CHECK_METHOD_IN_BAND_BFD"; case HEALTH_CHECK_METHOD_BFD: return "HEALTH_CHECK_METHOD_BFD"; case HEALTH_CHECK_METHOD_HTTP: return "HEALTH_CHECK_METHOD_HTTP"; default: return NULL; } } // return 0 : success // return -1 : key not exist int health_check_session_get_mac(uint64_t session_id, char *mac_buff) { uint8_t *p = NULL; const char *str_method = NULL; struct session_iterm *tmp = NULL; uint8_t mac[HC_MAC_LEN] = {0}; uint8_t init_mac[HC_MAC_LEN] = {0}; if (enable == 0) { return 0; } pthread_rwlock_rdlock(&g_handle.rwlock); HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp); if (!tmp) { LOG_DEBUG("health check session get mac: session id [%lu] not exists", session_id); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } str_method = health_check_method_str(tmp->policy.method); if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD && tmp->is_active == 0) { LOG_DEBUG("health check session id [%lu] profile id [%d] health check method [%s] active is down", session_id, tmp->profile_id, str_method); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD) { health_check_method_table_get_mac(&g_handle_bfd, tmp->policy.address, mac); } else if (tmp->policy.method == HEALTH_CHECK_METHOD_NONE) { health_check_method_table_get_mac(&g_handle_none, tmp->policy.address, mac); } if (memcmp(mac, init_mac, HC_MAC_LEN) == 0) { health_check_method_table_get_mac(&g_handle_none, gateway_address, mac); if (memcmp(mac, init_mac, HC_MAC_LEN) == 0) { LOG_DEBUG("health check session id [%lu] profile id [%d] health check method [%s] get mac [null]", session_id, tmp->profile_id, str_method); pthread_rwlock_unlock(&g_handle.rwlock); return -1; } } p = mac; snprintf(mac_buff, 18, "%02x:%02x:%02x:%02x:%02x:%02x", p[0], p[1], p[2], p[3], p[4], p[5]); LOG_DEBUG("health check session id [%lu] profile id [%d] health check method [%s] get mac [%s]", session_id, tmp->profile_id, str_method, mac_buff); pthread_rwlock_unlock(&g_handle.rwlock); return 0; }