tango-tsg-service-chaining-engine/platform/src/health_check.cpp

#include <string.h>
#include <pthread.h>
#include <assert.h>
#include <unistd.h>
#include <MESA/MESA_prof_load.h>
#include <net/if.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/if_ether.h>
#include <net/if_arp.h>
#include <arpa/inet.h>
#include <errno.h>

#include "log.h"
#include "uthash.h"
#include "bfd.h"
#include "sf_status.h"
#include "health_check.h"


#define     HC_MAC_LEN              6
#define     HC_DEV_NAME_LEN         16
#define     HC_LOCAL_ADDRESS_LEN    64

struct session_table
{
    // handler;
    struct session_iterm *root_by_id;
    pthread_rwlock_t rwlock;
};

struct session_iterm
{
    uint64_t session_id;                // key

    struct health_check policy;         // value1: deep copy
    int is_active;                      // value2
    int profile_id;                     // value3
    uint8_t mac[HC_MAC_LEN];            // value4
    int vsys_id;                        // value5

    UT_hash_handle hh1; /* handle for first hash table */
};

struct session_table_addr
{
    // handler;
    struct node_addr *htable;
};

struct node_addr
{
    char address[64]; // key

    uint64_t session_id;        // session id
    int ref_cnt;                // reference 

    UT_hash_handle hh; /* handle for first hash table */
};

static uint64_t g_session_id;
static struct session_table g_handle;
static struct session_table_addr g_handle_addr;
static struct sf_status *g_sf_status = NULL;

int sleep_ms = 300;
int enable = 1;
char path[BFD_PATHLEN];
char hc_dev_name[HC_DEV_NAME_LEN];
char local_address[HC_LOCAL_ADDRESS_LEN];
char gateway_address[HC_LOCAL_ADDRESS_LEN];
uint8_t default_gw_mac[HC_MAC_LEN];

static int get_mac_by_addr(char *addr, uint8_t *buf);
static int health_check_session_foreach();

static void health_check_ref_inc(struct node_addr *node)
{
    node->ref_cnt++;
}

static int health_check_ref_dec(struct node_addr *node)
{
    node->ref_cnt--;
    return node->ref_cnt;
}

static struct session_iterm *health_check_get_iterm_by_id(uint64_t session_id)
{
    struct session_iterm *tmp = NULL;
    pthread_rwlock_rdlock(&g_handle.rwlock);
    HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp);
    pthread_rwlock_unlock(&g_handle.rwlock);
    return tmp;
}

void health_check_session_init(const char *profile)
{
    memset(&g_handle, 0, sizeof(g_handle));
    pthread_rwlock_init(&g_handle.rwlock, NULL);
    MESA_load_profile_int_def(profile, "bfdd", "enable", &enable, 1);
    MESA_load_profile_string_def(profile, "bfdd", "path", path, sizeof(path), "/var/run/frr/bfdd.vty");
    MESA_load_profile_string_def(profile, "bfdd", "device", hc_dev_name, sizeof(hc_dev_name), "eth0");
    MESA_load_profile_string_def(profile, "bfdd", "local_address", local_address, sizeof(local_address), "127.0.0.1");
    MESA_load_profile_string_def(profile, "bfdd", "gateway", gateway_address, sizeof(gateway_address), "127.0.0.1");

    if (enable == 0)
    {
        return;
    }

    g_sf_status = sf_status_create(profile);
    // TODO: 循环获取？
    get_mac_by_addr(gateway_address, default_gw_mac);
    health_check_session_foreach();
}

static int health_check_session_recover_cfg(struct bfd_vtysh_client *client)
{
    int ret = 0;
    struct session_iterm *tmp = NULL;
    struct session_iterm *node = NULL;
    HASH_ITER(hh1, g_handle.root_by_id, node, tmp) {
        if (node->policy.method != HEALTH_CHECK_METHOD_BFD)
            continue;

        ret = bfd_vtysh_add_dev(client, node->policy.address, node->policy.retires, node->policy.interval_ms);
        if (ret != 0)
            return -1;
    }
    return 0;
}

static void health_check_session_init_bfd_client(struct bfd_vtysh_client *client)
{
    memset(client, 0, sizeof(*client));
    snprintf(client->path, sizeof(client->path), path);
    client->pre_config = bfd_vtysh_pre_config;
}

static void health_check_session_recover_bfd(struct bfd_vtysh_client *client)
{
    memset(client, 0, sizeof(*client));
    snprintf(client->path, sizeof(client->path), path);
    client->pre_config = bfd_vtysh_pre_config;
    client->recover_config = health_check_session_recover_cfg;
}

static struct node_addr *health_check_get_node_by_addr(const char *addr)
{
    struct node_addr *tmp = NULL;
    HASH_FIND_STR(g_handle_addr.htable, addr, tmp);
    return tmp;
}

static void health_check_add_node_by_addr(struct node_addr *node, const char *addr, uint64_t session_id)
{
    snprintf(node->address, sizeof(node->address), addr);
    node->session_id = session_id;
    health_check_ref_inc(node);
    HASH_ADD_STR(g_handle_addr.htable, address, node);
}

static int health_check_del_node_by_addr(const char *addr)
{
    int ret = 0;
    struct node_addr *node = NULL;

    node = health_check_get_node_by_addr(addr);
    if (node == NULL)
        return 0;

    ret = health_check_ref_dec(node);
    if (ret != 0)
        return 1;

    HASH_DEL(g_handle_addr.htable, node);
    free(node);
    node = NULL;
    return 0;
}

static uint64_t health_check_get_session_id()
{
    struct session_iterm *tmp = NULL;
    uint64_t tmp_session_id = g_session_id;

    while(1) {
        g_session_id++;
        if (g_session_id == 0)
            g_session_id++;
        if (tmp_session_id == g_session_id)
            return 0;
        tmp = health_check_get_iterm_by_id(g_session_id);
        if (tmp)
            continue;
        break;
    }
    return g_session_id;
}

// return  >0 : session id
// return  0 : fail
// struct health_check *policy : need deep copy
uint64_t health_check_session_add(int profile_id, int vsys_id, const struct health_check *policy)
{
    int ret = 0;
    uint64_t session_id = 0;
    struct bfd_vtysh_client client;
    struct node_addr *node = NULL;
    struct session_iterm *tmp = NULL;

    if (enable == 0)
    {
        return 1;
    }
    session_id = health_check_get_session_id();
    if (session_id == 0) {
        LOG_ERROR("health check get session id failed!");
        return 0;
    }

    tmp = (struct session_iterm *)calloc(1, sizeof(struct session_iterm));
    assert(tmp);

    tmp->vsys_id = vsys_id;
    tmp->session_id = session_id;
    tmp->profile_id = profile_id;
    memcpy(&tmp->policy, policy, sizeof(struct health_check));
    
    pthread_rwlock_wrlock(&g_handle.rwlock);
    HASH_ADD(hh1, g_handle.root_by_id, session_id, sizeof(tmp->session_id), tmp);
    pthread_rwlock_unlock(&g_handle.rwlock);

    if (policy->method == HEALTH_CHECK_METHOD_BFD) {
        node = health_check_get_node_by_addr(policy->address);
        if (node) {
            health_check_ref_inc(node);
        }
        else {
            node = (struct node_addr *)calloc(1, sizeof(struct node_addr));
            assert(node);
            health_check_add_node_by_addr(node, policy->address, session_id);
        }
        health_check_session_init_bfd_client(&client);
        bfd_vtysh_connect(&client);
        ret = bfd_vtysh_add_dev(&client, policy->address, policy->retires, policy->interval_ms);
        if (ret != 0)
            LOG_ERROR("bfd vtysh add dev address [%s] failed!", policy->address);
        bfd_vtysh_close(&client);
    }

    LOG_DEBUG("health check session table insert: profile id [%d] session id [%lu] address [%s] success", profile_id, session_id, policy->address);
    return session_id;
}

// return  0 : success
// return -1 : key not exist
int health_check_session_del(uint64_t session_id, int profile_id)
{
    int ret = 0;
    struct bfd_vtysh_client client;
    struct session_iterm *tmp = NULL;

    if (enable == 0)
    {
        return 0;
    }

    tmp = health_check_get_iterm_by_id(session_id);
    if (!tmp) {
        LOG_DEBUG("health check session table delete: session id [%lu] not exists", session_id);
        return -1;
    }

    if (tmp->policy.method == HEALTH_CHECK_METHOD_BFD) {
        ret = health_check_del_node_by_addr(tmp->policy.address);
        if (ret == 1)
            goto end;

        health_check_session_init_bfd_client(&client);
        bfd_vtysh_connect(&client);
        ret = bfd_vtysh_del_dev(&client, tmp->policy.address);
        if (ret != 0) {
            LOG_ERROR("bfd vtysh delete dev address [%s] failed!", tmp->policy.address);
        }
        bfd_vtysh_close(&client);
    }

end:
    pthread_rwlock_wrlock(&g_handle.rwlock);
    HASH_DELETE(hh1, g_handle.root_by_id, tmp);
    sf_status_delete(g_sf_status, profile_id);
    pthread_rwlock_unlock(&g_handle.rwlock);
    free(tmp);
    tmp = NULL;

    LOG_DEBUG("health check session table delete: session id [%lu] success", session_id);
    return 0;
}

// return  1 : active
// return  0 : inactive
// return -1 : key not exist
int health_check_session_get_status(uint64_t session_id)
{
    int status = 0;
    struct session_iterm *tmp = NULL;

    if (enable == 0)
    {
        return 1;
    }

    pthread_rwlock_rdlock(&g_handle.rwlock);
    HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp);
    if (!tmp) {
        LOG_DEBUG("health check session table get status: session id [%lu] not exists", session_id);
        pthread_rwlock_unlock(&g_handle.rwlock);
        return -1;
    }

    status = tmp->is_active;
    pthread_rwlock_unlock(&g_handle.rwlock);
    LOG_DEBUG("health check session id[%lu] get status [%d]", session_id, status);
    return status;
}

// return  0 : success
// return -1 : key not exist
int health_check_session_set_status(uint64_t session_id, int is_active)
{
    struct session_iterm *tmp = NULL;

    if (enable == 0)
    {
        return 0;
    }

    pthread_rwlock_wrlock(&g_handle.rwlock);
    HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp);
    if (!tmp) {
        LOG_DEBUG("health check session table set status: session id [%lu] not exists", session_id);
        pthread_rwlock_unlock(&g_handle.rwlock);
        return -1;
    }

    tmp->is_active = is_active;
    pthread_rwlock_unlock(&g_handle.rwlock);
    return 0;
}

static int get_mac_by_addr(char *addr, uint8_t *buf)
{
    int sfd, ret;
    struct arpreq arp_req;
    struct sockaddr_in *sin;

    sin = (struct sockaddr_in *)&(arp_req.arp_pa);

    memset(&arp_req, 0, sizeof(arp_req));
    sin->sin_family = AF_INET;
    inet_pton(AF_INET, addr, &(sin->sin_addr));
    snprintf(arp_req.arp_dev, IFNAMSIZ, hc_dev_name);

    sfd = socket(AF_INET, SOCK_DGRAM, 0);
    if (sfd == -1)
        return -1;

    ret = ioctl(sfd, SIOCGARP, &arp_req);
    if (ret < 0)
        memcpy(buf, default_gw_mac, HC_MAC_LEN);
    else
        memcpy(buf, arp_req.arp_ha.sa_data, HC_MAC_LEN);

    LOG_DEBUG("IP:%s, MAC: %02x:%02x:%02x:%02x:%02x:%02x",
                addr, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
    close(sfd);
    return 0;
}

static void *_health_check_session_foreach(void *arg)
{
    int is_active = 0;
    int interval_s = sf_status_get_interval(g_sf_status);
    struct bfd_vtysh_client client;
    struct session_iterm *tmp = NULL;
    struct session_iterm *node = NULL;

    struct timespec current_time;
    struct timespec g_status_last_send_time;

    clock_gettime(CLOCK_MONOTONIC, &current_time);
    clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time);

    health_check_session_init_bfd_client(&client);
    bfd_vtysh_connect(&client);

    while(1) {
        // TODO: 改为读锁，更新数据入队列，通过写锁更新
        pthread_rwlock_wrlock(&g_handle.rwlock);
        HASH_ITER(hh1, g_handle.root_by_id, node, tmp) {
            if (node->policy.method != HEALTH_CHECK_METHOD_BFD)
                continue;

            if (strlen(node->policy.address) != 0) {
                is_active = bfd_vtysh_get_dev_active(&client, node->policy.address);
                if (is_active == -1) {
                    bfd_vtysh_close(&client);
                    health_check_session_recover_bfd(&client);
                    bfd_vtysh_connect(&client);
                    is_active = bfd_vtysh_get_dev_active(&client, node->policy.address);
                    if (is_active == -1)
                        is_active = 0;
                }
            }
            else {
                is_active = 0;
            }

            sf_status_update(g_sf_status, node->profile_id, node->vsys_id, is_active, 0);
            if (node->is_active != is_active) {
                node->is_active = is_active;
                if (node->is_active == 1) {
                    get_mac_by_addr(node->policy.address, node->mac);
                }
                else {
                    memset(node->mac, 0, sizeof(node->mac));
                }
            }
            if (sleep_ms > node->policy.interval_ms)
                sleep_ms = node->policy.interval_ms;
        }
        pthread_rwlock_unlock(&g_handle.rwlock);

        clock_gettime(CLOCK_MONOTONIC, &current_time);
        if (current_time.tv_sec - g_status_last_send_time.tv_sec >= interval_s)
        {
            sf_status_send(g_sf_status);
            clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time);
        }

        // interval_s : 1000 ms
        // sleep_ms   :  900 ms
        if (interval_s * 1000 > sleep_ms)
        {
            usleep(sleep_ms * 1000);
        }
        // interval_s :  900 ms
        // sleep_ms   : 1000 ms
        else
        {
            usleep(interval_s * 1000 * 1000);

            clock_gettime(CLOCK_MONOTONIC, &current_time);
            if (current_time.tv_sec - g_status_last_send_time.tv_sec >= interval_s)
            {
                sf_status_send(g_sf_status);
                clock_gettime(CLOCK_MONOTONIC, &g_status_last_send_time);
            }
            usleep(sleep_ms * 1000 - interval_s * 1000 * 1000);
        }
    }
    bfd_vtysh_close(&client);
    return NULL;
}

static int health_check_session_foreach()
{    
    pthread_t pid;
    pthread_create(&pid, NULL, _health_check_session_foreach, NULL);
    pthread_detach(pid);
    return pid;
}

// return  0 : success
// return -1 : key not exist
int health_check_session_get_mac(uint64_t session_id, char *mac_buff)
{
    uint8_t *p = NULL;
    struct session_iterm *tmp = NULL;
    uint8_t init_mac[HC_MAC_LEN] = {0};

    if (enable == 0)
    {
        return 0;
    }
    pthread_rwlock_rdlock(&g_handle.rwlock);
    HASH_FIND(hh1, g_handle.root_by_id, &session_id, sizeof(session_id), tmp);
    if (!tmp) {
        LOG_DEBUG("health check session get mac: session id [%lu] not exists", session_id);
        pthread_rwlock_unlock(&g_handle.rwlock);
        return -1;
    }

    p = (uint8_t *)tmp->mac;
    if (memcmp(p, init_mac, HC_MAC_LEN) == 0) {
        pthread_rwlock_unlock(&g_handle.rwlock);
        return -1;
    }
    snprintf(mac_buff, 18, "%02x:%02x:%02x:%02x:%02x:%02x", p[0], p[1], p[2], p[3], p[4], p[5]);
    pthread_rwlock_unlock(&g_handle.rwlock);
    LOG_DEBUG("health check session id [%lu] get mac [%s]", session_id, mac_buff);

    return 0;
}