From a43d1d04e82e90ede42aa245a2b3f914e7e689e8 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 14 Feb 2019 17:27:16 -0800 Subject: [PATCH] Unfinished callback improvements, consolidated netif4 and netif6 --- Makefile | 2 +- include/libzt.h | 179 +++++++++++++++++++++++++--- src/Controls.cpp | 79 ++++++++++--- src/Controls.hpp | 27 ++++- src/Options.h | 5 - src/Service.cpp | 64 ++++++---- src/VirtualTap.cpp | 97 ++++++++------- src/VirtualTap.hpp | 44 +++---- src/lwipDriver.cpp | 285 ++++++++++++++++++++++++--------------------- 9 files changed, 515 insertions(+), 267 deletions(-) diff --git a/Makefile b/Makefile index bc0ca4e..4d465fe 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ patch: .PHONY: clean clean: - rm -rf bin staging generated + rm -rf bin staging generated dist all: debug release diff --git a/include/libzt.h b/include/libzt.h index 950ce60..149d649 100644 --- a/include/libzt.h +++ b/include/libzt.h @@ -33,6 +33,8 @@ #ifndef LIBZT_H #define LIBZT_H +#include "lwipopts.h" + #ifdef _WIN32 #ifdef ADD_EXPORTS #define ZT_SOCKET_API __declspec(dllexport) @@ -113,10 +115,9 @@ typedef int zts_err_t; #define ZTS_EVENT_NODE_ONLINE 2 #define ZTS_EVENT_NODE_DOWN 3 #define ZTS_EVENT_NODE_IDENTITY_COLLISION 4 -// libzt node events #define ZTS_EVENT_NODE_UNRECOVERABLE_ERROR 16 #define ZTS_EVENT_NODE_NORMAL_TERMINATION 17 -// Network-specific events +// Network events #define ZTS_EVENT_NETWORK_NOT_FOUND 32 #define ZTS_EVENT_NETWORK_CLIENT_TOO_OLD 33 #define ZTS_EVENT_NETWORK_REQUESTING_CONFIG 34 @@ -125,23 +126,41 @@ typedef int zts_err_t; #define ZTS_EVENT_NETWORK_READY_IP4 37 #define ZTS_EVENT_NETWORK_READY_IP6 38 #define ZTS_EVENT_NETWORK_DOWN 39 -// -#define ZTS_EVENT_NETWORK_STACK_UP 48 -#define ZTS_EVENT_NETWORK_STACK_DOWN 49 - +// Network Stack events +#define ZTS_EVENT_STACK_UP 48 +#define ZTS_EVENT_STACK_DOWN 49 // lwIP netif events -#define ZTS_EVENT_NETIF_UP_IP4 64 -#define ZTS_EVENT_NETIF_UP_IP6 65 -#define ZTS_EVENT_NETIF_DOWN_IP4 66 -#define ZTS_EVENT_NETIF_DOWN_IP6 67 -#define ZTS_EVENT_NETIF_REMOVED 68 -#define ZTS_EVENT_NETIF_LINK_UP 69 -#define ZTS_EVENT_NETIF_LINK_DOWN 70 -#define ZTS_EVENT_NETIF_NEW_ADDRESS 71 +#define ZTS_EVENT_NETIF_UP 64 +#define ZTS_EVENT_NETIF_DOWN 65 +#define ZTS_EVENT_NETIF_REMOVED 66 +#define ZTS_EVENT_NETIF_LINK_UP 67 +#define ZTS_EVENT_NETIF_LINK_DOWN 68 // Peer events #define ZTS_EVENT_PEER_P2P 96 #define ZTS_EVENT_PEER_RELAY 97 #define ZTS_EVENT_PEER_UNREACHABLE 98 +// Path events +#define ZTS_EVENT_PATH_DISCOVERED 112 +#define ZTS_EVENT_PATH_ALIVE 113 +#define ZTS_EVENT_PATH_DEAD 114 +// Route events +#define ZTS_EVENT_ROUTE_ADDED 128 +#define ZTS_EVENT_ROUTE_REMOVED 129 +// Address events +#define ZTS_EVENT_ADDR_ADDED_IP4 144 +#define ZTS_EVENT_ADDR_REMOVED_IP4 145 +#define ZTS_EVENT_ADDR_ADDED_IP6 146 +#define ZTS_EVENT_ADDR_REMOVED_IP6 147 + +// Macros for legacy behaviour +#define NODE_EVENT_TYPE(code) code >= ZTS_EVENT_NODE_UP && code <= ZTS_EVENT_NODE_NORMAL_TERMINATION +#define NETWORK_EVENT_TYPE(code) code >= ZTS_EVENT_NETWORK_NOT_FOUND && code <= ZTS_EVENT_NETWORK_DOWN +#define STACK_EVENT_TYPE(code) code >= ZTS_EVENT_STACK_UP && code <= ZTS_EVENT_STACK_DOWN +#define NETIF_EVENT_TYPE(code) code >= ZTS_EVENT_NETIF_UP && code <= ZTS_EVENT_NETIF_LINK_DOWN +#define PEER_EVENT_TYPE(code) code >= ZTS_EVENT_PEER_P2P && code <= ZTS_EVENT_PEER_UNREACHABLE +#define PATH_EVENT_TYPE(code) code >= ZTS_EVENT_PATH_DISCOVERED && code <= ZTS_EVENT_PATH_DEAD +#define ROUTE_EVENT_TYPE(code) code >= ZTS_EVENT_ROUTE_ADDED && code <= ZTS_EVENT_ROUTE_REMOVED +#define ADDR_EVENT_TYPE(code) code >= ZTS_EVENT_ADDR_ADDED_IP4 && code <= ZTS_EVENT_ADDR_REMOVED_IP6 ////////////////////////////////////////////////////////////////////////////// // Common definitions and structures for interacting with the ZT socket API // @@ -287,7 +306,7 @@ struct zts_in6_addr { u32_t u32_addr[4]; u8_t u8_addr[16]; } un; -#define s6_addr un.u8_addr +//#define s6_addr un.u8_addr }; struct zts_sockaddr_in { @@ -416,6 +435,134 @@ enum zts_peer_role ZTS_PEER_ROLE_PLANET = 2 // planetary root }; +/** + * A structure used to convey details about the current node + * to the user application + */ +struct zts_node_details +{ + /** + * The node ID + */ + uint64_t address; + + /** + * The current clock value accord to the node + */ + uint64_t clock; + + /** + * Whether or not this node is online + */ + bool online; + + /** + * Whether port mapping is enabled + */ + bool portMappingEnabled; + + /** + * Whether multipath support is enabled. If true, this node will + * be capable of utilizing multiple physical links simultaneosly + * to create higher quality or more robust aggregate links. + * + * See: https://www.zerotier.com/manual.shtml#2_1_5 + */ + bool multipathEnabled; + + /** + * The port used by the service to send and receive + * all encapsulated traffic + */ + uint16_t primaryPort; + + /** + * Planet ID + */ + uint64_t planetWorldId; + uint64_t planetWorldTimestamp; + uint8_t versionMajor; + uint8_t versionMinor; + uint8_t versionRev; +}; + +/** + * A structure used to convey information to a user application via + * a callback function. + */ +struct zts_callback_msg +{ + zts_callback_msg(): + eventCode(-1), + node(NULL), + network(NULL), + netif(NULL), + route(NULL), + path(NULL), + peer(NULL) {} + + /** + * Event identifier + */ + int eventCode; + + struct zts_node_details *node; + struct zts_network_details *network; + struct zts_netif_details *netif; + struct zts_virtual_network_route *route; + struct zts_physical_path *path; + struct zts_peer_details *peer; + struct zts_addr_details *addr; +}; + +struct zts_addr_details +{ + uint64_t nwid; + struct sockaddr_storage addr; +}; + +/** + * A structure used to convey information about a virtual network + * interface (netif) to a user application. + */ +struct zts_netif_details +{ + /** + * The virtual network that this interface was commissioned for. + */ + uint64_t nwid; + + /** + * The hardware address assigned to this interface + */ + uint64_t mac; + + /** + * The MTU for this interface + */ + int mtu; + + /** + * The IPv4 address assigned to this interface. + */ + //struct sockaddr_in ip4_addr; + + /** + * The IPv6 addresses assigned to this interface. + */ + //struct sockaddr_in6 ip6_addr[LWIP_IPV6_NUM_ADDRESSES]; + + /** + * Number of IPv4 addresses assigned to this interface + */ + //int num_ip4_addr; + + /** + * Number of IPv6 addresses assigned to this interface + */ + //int num_ip6_addr; +}; + /** * A structure used to represent a virtual network route */ @@ -590,7 +737,7 @@ extern "C" { * @param userCallbackFunc User-specified callback for ZeroTier events * @return 0 if successful; or 1 if failed */ -ZT_SOCKET_API int ZTCALL zts_start(const char *path, void (*userCallbackFunc)(uint64_t, int), int port = ZTS_DEFAULT_PORT); +ZT_SOCKET_API int ZTCALL zts_start(const char *path, void (*userCallbackFunc)(struct zts_callback_msg*), int port = ZTS_DEFAULT_PORT); /** * @brief Stops the ZeroTier service, brings down all virtual interfaces in order to stop all traffic processing. diff --git a/src/Controls.cpp b/src/Controls.cpp index 7531d23..15444b2 100644 --- a/src/Controls.cpp +++ b/src/Controls.cpp @@ -113,40 +113,81 @@ OneService *service; static jmethodID _userCallbackMethodRef = NULL; #endif -void (*_userEventCallbackFunc)(uint64_t, int); +void (*_userEventCallbackFunc)(struct zts_callback_msg *); -extern moodycamel::ConcurrentQueue*> _callbackMsgQueue; +moodycamel::ConcurrentQueue _callbackMsgQueue; ////////////////////////////////////////////////////////////////////////////// // Internal ZeroTier Service Controls (user application shall not use these)// ////////////////////////////////////////////////////////////////////////////// -void postEvent(uint64_t id, int eventCode) +void postEvent(int eventCode, void *arg) { - // Queue callback event messages from other threads (such as lwIP driver) - _callbackMsgQueue.enqueue(new std::pair(id, eventCode)); + struct zts_callback_msg *msg = new zts_callback_msg(); + msg->eventCode = eventCode; + + if (NODE_EVENT_TYPE(eventCode)) { + msg->node = (struct zts_node_details*)arg; + } if (NETWORK_EVENT_TYPE(eventCode)) { + msg->network = (struct zts_network_details*)arg; + } if (NETIF_EVENT_TYPE(eventCode)) { + msg->netif = (struct zts_netif_details*)arg; + } if (ROUTE_EVENT_TYPE(eventCode)) { + msg->route = (struct zts_virtual_network_route*)arg; + } if (PATH_EVENT_TYPE(eventCode)) { + msg->path = (struct zts_physical_path*)arg; + } if (PEER_EVENT_TYPE(eventCode)) { + msg->peer = (struct zts_peer_details*)arg; + } if (ADDR_EVENT_TYPE(eventCode)) { + msg->addr = (struct zts_addr_details*)arg; + } + _callbackMsgQueue.enqueue(msg); } -void _process_callback_event_helper(uint64_t nwid, int eventCode) +void postEvent(int eventCode) { + postEvent(eventCode, (void*)0); +} + +void freeEvent(struct zts_callback_msg *msg) +{ + if (msg->node) { delete msg->node; } + if (msg->network) { delete msg->network; } + if (msg->netif) { delete msg->netif; } + if (msg->route) { delete msg->route; } + if (msg->path) { delete msg->path; } + if (msg->peer) { delete msg->peer; } + if (msg->addr) { delete msg->addr; } +} + +void _process_callback_event_helper(struct zts_callback_msg *msg) { #ifdef SDK_JNI if(_userCallbackMethodRef) { JNIEnv *env; jint rs = jvm->AttachCurrentThread(&env, NULL); assert (rs == JNI_OK); - env->CallVoidMethod(objRef, _userCallbackMethodRef, nwid, eventCode); + if (NODE_EVENT_TYPE(msg->eventCode)) { + arg = msg->networkId; + } + if (NODE_EVENT_TYPE(msg->eventCode)) { + arg = msg->nodeId; + } + if (NODE_EVENT_TYPE(msg->eventCode)) { + arg = msg->nodeId; + } + env->CallVoidMethod(objRef, _userCallbackMethodRef, msg->networkId, msg->eventCode); } #else if (_userEventCallbackFunc) { - _userEventCallbackFunc(nwid, eventCode); + _userEventCallbackFunc(msg); } #endif } -void _process_callback_event(uint64_t nwid, int eventCode) +void _process_callback_event(struct zts_callback_msg *msg) { _callback_lock.lock(); - _process_callback_event_helper(nwid, eventCode); + _process_callback_event_helper(msg); _callback_lock.unlock(); } @@ -215,11 +256,11 @@ void *_zts_run_callbacks(void *thread_id) #endif while (_run_callbacks || _callbackMsgQueue.size_approx() > 0) { - std::pair *msg; - for (int j = 0; j != 32; j++) { // TODO: Check size of queue + struct zts_callback_msg *msg; + int sz = _callbackMsgQueue.size_approx(); + for (int j = 0; j < sz; j++) { if (_callbackMsgQueue.try_dequeue(msg)) { - // DEBUG_INFO("deqeueuing front: %llx,%d", msg->first, msg->second); - _process_callback_event(msg->first, msg->second); + _process_callback_event(msg); delete msg; } } @@ -272,12 +313,12 @@ void *_zts_run_service(void *arg) switch(service->run()) { case OneService::ONE_STILL_RUNNING: case OneService::ONE_NORMAL_TERMINATION: - postEvent((uint64_t)0, ZTS_EVENT_NODE_NORMAL_TERMINATION); + postEvent(ZTS_EVENT_NODE_NORMAL_TERMINATION); break; case OneService::ONE_UNRECOVERABLE_ERROR: DEBUG_ERROR("fatal error: %s", service->fatalErrorMessage().c_str()); err = true; - postEvent((uint64_t)0, ZTS_EVENT_NODE_UNRECOVERABLE_ERROR); + postEvent(ZTS_EVENT_NODE_UNRECOVERABLE_ERROR); break; case OneService::ONE_IDENTITY_COLLISION: { err = true; @@ -290,7 +331,7 @@ void *_zts_run_service(void *arg) OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str()); OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.public").c_str()); } - postEvent((uint64_t)0, ZTS_EVENT_NODE_IDENTITY_COLLISION); + postEvent(ZTS_EVENT_NODE_IDENTITY_COLLISION); } continue; // restart! } break; // terminate loop -- normally we don't keep restarting @@ -300,7 +341,7 @@ void *_zts_run_service(void *arg) delete service; service = (OneService *)0; _service_lock.unlock(); - postEvent((uint64_t)0, ZTS_EVENT_NODE_DOWN); + postEvent(ZTS_EVENT_NODE_DOWN); } catch ( ... ) { DEBUG_ERROR("unexpected exception starting ZeroTier instance"); } @@ -411,7 +452,7 @@ zts_err_t zts_deorbit(uint64_t moonWorldId) #ifdef SDK_JNI #endif -zts_err_t zts_start(const char *path, void (*callback)(uint64_t, int), int port) +zts_err_t zts_start(const char *path, void (*callback)(struct zts_callback_msg*), int port) { Mutex::Lock _l(_service_lock); lwip_driver_init(); diff --git a/src/Controls.hpp b/src/Controls.hpp index a60112d..4b3870e 100644 --- a/src/Controls.hpp +++ b/src/Controls.hpp @@ -39,7 +39,32 @@ namespace ZeroTier { // ZeroTier Internal Service Controls // ////////////////////////////////////////////////////////////////////////////// -void postEvent(uint64_t id, int eventCode); +/** + * Add a callback event message to the queue. This can be safely called + * from other threads since a lock-free queue is used. + * + * @param eventCode The event ID for this event + * @param msg Pointer to a structure of pointers to other message-relevant + * data structures. + */ +void postEvent(int eventCode, void *arg); + +/** + * Add a callback event message to the queue. This can be safely called + * from other threads since a lock-free queue is used. Note: For use in + * situations when no additional information needs to be conveyed to the + * user application. + * + * @param eventCode The event ID for this event + */ +void postEvent(int eventCode); + +/** + * Free whatever was allocated to contain the callback message + * + * @param msg Message to be freed + */ +void freeEvent(struct zts_callback_msg *msg); #ifdef __cplusplus extern "C" { diff --git a/src/Options.h b/src/Options.h index 569c871..076f1c8 100644 --- a/src/Options.h +++ b/src/Options.h @@ -5,11 +5,6 @@ // Callbacks // ////////////////////////////////////////////////////////////////////////////// -#define ZTS_NODE_CALLBACKS 1 -#define ZTS_NETWORK_CALLBACKS 1 -#define ZTS_NETIF_CALLBACKS 1 -#define ZTS_PEER_CALLBACKS 1 - /** * The maximum number of un-processed callback messages */ diff --git a/src/Service.cpp b/src/Service.cpp index a23315d..36d4db3 100644 --- a/src/Service.cpp +++ b/src/Service.cpp @@ -86,6 +86,9 @@ #include #endif +#include "libzt.h" +#include "Controls.hpp" + // Use the virtual netcon endpoint instead of a tun/tap port driver #include "VirtualTap.hpp" namespace ZeroTier { typedef VirtualTap EthernetTap; } @@ -102,8 +105,7 @@ namespace ZeroTier { typedef VirtualTap EthernetTap; } namespace ZeroTier { -// Concurrent queue for callback message processing -moodycamel::ConcurrentQueue*> _callbackMsgQueue; +extern void postEvent(uint64_t id, int eventCode); namespace { @@ -803,7 +805,14 @@ public: { // Feed node events into lock-free queue for later dequeuing by the callback thread if (event <= ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION) { - _callbackMsgQueue.enqueue(new std::pair(0x0000000000, event)); + if (event == ZTS_EVENT_NODE_ONLINE) { + struct zts_node_details *nd = new zts_node_details; + nd->address = _node->address(); + postEvent(event, (void*)nd); + } + else { + postEvent(event, (void*)0); + } } switch(event) { case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION: { @@ -827,11 +836,12 @@ public: inline void generateEventMsgs() { - if (!lwip_is_up()) { - return; // Don't process peer status events unless the stack is up. + // Force the ordering of callback messages, these messages are + // only useful if the node and stack are both up and running + if (!_node->online() || !lwip_is_up()) { + return; } // Generate messages to be dequeued by the callback message thread -#if ZTS_NETWORK_CALLBACKS Mutex::Lock _l(_nets_m); for(std::map::iterator n(_nets.begin());n!=_nets.end();++n) { int mostRecentStatus = n->second.config.status; @@ -840,52 +850,63 @@ public: if (n->second.tap->_networkStatus == mostRecentStatus) { continue; // No state change } + struct zts_network_details *nd = new zts_network_details; + + //memcpy(nd, &(pl->peers[i]), sizeof(struct zts_network_details)); + nd->nwid = nwid; + switch (mostRecentStatus) { case ZT_NETWORK_STATUS_NOT_FOUND: - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_NOT_FOUND)); + postEvent(ZTS_EVENT_NETWORK_NOT_FOUND, (void*)nd); break; case ZT_NETWORK_STATUS_CLIENT_TOO_OLD: - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_CLIENT_TOO_OLD)); + postEvent(ZTS_EVENT_NETWORK_CLIENT_TOO_OLD, (void*)nd); break; case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION: - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_REQUESTING_CONFIG)); + postEvent(ZTS_EVENT_NETWORK_REQUESTING_CONFIG, (void*)nd); break; case ZT_NETWORK_STATUS_OK: - if (tap->netif4 && lwip_is_netif_up(tap->netif4)) { - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_READY_IP4)); + if (tap->hasIpv4Addr() && lwip_is_netif_up(tap->netif)) { + postEvent(ZTS_EVENT_NETWORK_READY_IP4, (void*)nd); } - if (tap->netif6 && lwip_is_netif_up(tap->netif6)) { - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_READY_IP6)); + if (tap->hasIpv6Addr() && lwip_is_netif_up(tap->netif)) { + postEvent(ZTS_EVENT_NETWORK_READY_IP6, (void*)nd); } - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_OK)); + // In addition to the READY messages, send one OK message + postEvent(ZTS_EVENT_NETWORK_OK, (void*)nd); break; case ZT_NETWORK_STATUS_ACCESS_DENIED: - _callbackMsgQueue.enqueue(new std::pair(nwid, ZTS_EVENT_NETWORK_ACCESS_DENIED)); + postEvent(ZTS_EVENT_NETWORK_ACCESS_DENIED, (void*)nd); break; default: break; } n->second.tap->_networkStatus = mostRecentStatus; } -#endif // ZTS_NETWORK_CALLBACKS -#if ZTS_PEER_CALLBACKS + // TODO: Add ZTS_EVENT_PEER_NEW ZT_PeerList *pl = _node->peers(); if (pl) { for(unsigned long i=0;ipeerCount;++i) { + + struct zts_peer_details *pd = new zts_peer_details; + + memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details)); + // pl->peers[i].address, 0, 0, 0, NULL, 0); + if (!peerCache.count(pl->peers[i].address)) { if (pl->peers[i].pathCount > 0) { - _callbackMsgQueue.enqueue(new std::pair(pl->peers[i].address, ZTS_EVENT_PEER_P2P)); + postEvent(ZTS_EVENT_PEER_P2P, (void*)pd); } if (pl->peers[i].pathCount == 0) { - _callbackMsgQueue.enqueue(new std::pair(pl->peers[i].address, ZTS_EVENT_PEER_RELAY)); + postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd); } } else { if (peerCache[pl->peers[i].address] == 0 && pl->peers[i].pathCount > 0) { - _callbackMsgQueue.enqueue(new std::pair(pl->peers[i].address, ZTS_EVENT_PEER_P2P)); + postEvent(ZTS_EVENT_PEER_P2P, (void*)pd); } if (peerCache[pl->peers[i].address] > 0 && pl->peers[i].pathCount == 0) { - _callbackMsgQueue.enqueue(new std::pair(pl->peers[i].address, ZTS_EVENT_PEER_RELAY)); + postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd); } } // Update our cache with most recently observed path count @@ -893,7 +914,6 @@ public: } } _node->freeQueryResult((void *)pl); -#endif // ZTS_PEER_CALLBACKS } inline int networkCount() diff --git a/src/VirtualTap.cpp b/src/VirtualTap.cpp index ac056b0..58eba34 100644 --- a/src/VirtualTap.cpp +++ b/src/VirtualTap.cpp @@ -48,7 +48,7 @@ namespace ZeroTier { class VirtualTap; extern OneService *service; -extern void postEvent(uint64_t id, int eventCode); +extern void postEvent(int eventCode, void *arg); /** * A virtual tap device. The ZeroTier core service creates one of these for each @@ -86,7 +86,9 @@ VirtualTap::VirtualTap( VirtualTap::~VirtualTap() { - postEvent(_nwid, ZTS_EVENT_NETWORK_DOWN); + struct zts_network_details *nd = new zts_network_details; + nd->nwid = _nwid; + postEvent(ZTS_EVENT_NETWORK_DOWN, (void*)nd); _run = false; ::write(_shutdownSignalPipe[1],"\0",1); _phy.whack(); @@ -96,34 +98,6 @@ VirtualTap::~VirtualTap() ::close(_shutdownSignalPipe[1]); } -uint64_t VirtualTap::recognizeLowerLevelInterfaceStateChange(void *n) -{ - if (!n) { - return ZTS_EVENT_NONE; - } - if (n == netif4) { - if (netif4WasUpLastCheck && !lwip_is_netif_up(netif4)) { - netif4WasUpLastCheck = false; - return ZTS_EVENT_NETIF_DOWN_IP4; - } - if (!netif4WasUpLastCheck && lwip_is_netif_up(netif4)) { - netif4WasUpLastCheck = true; - return ZTS_EVENT_NETIF_UP_IP4; - } - } - if (n == netif6) { - if (netif6WasUpLastCheck && !lwip_is_netif_up(netif6)) { - netif6WasUpLastCheck = false; - return ZTS_EVENT_NETIF_DOWN_IP6; - } - if (!netif6WasUpLastCheck && lwip_is_netif_up(netif6)) { - netif6WasUpLastCheck = true; - return ZTS_EVENT_NETIF_UP_IP6; - } - } - return ZTS_EVENT_NONE; -} - void VirtualTap::lastConfigUpdate(uint64_t lastConfigUpdateTime) { _lastConfigUpdateTime = lastConfigUpdateTime; @@ -139,13 +113,49 @@ bool VirtualTap::enabled() const return _enabled; } +bool VirtualTap::hasIpv4Addr() +{ + Mutex::Lock _l(_ips_m); + std::vector::iterator it(_ips.begin()); + while (it != _ips.end()) { + if ((*it).isV4()) { return true; } + it++; + } + return false; +} + +bool VirtualTap::hasIpv6Addr() +{ + Mutex::Lock _l(_ips_m); + std::vector::iterator it(_ips.begin()); + while (it != _ips.end()) { + if ((*it).isV6()) { return true; } + it++; + } + return false; +} + bool VirtualTap::addIp(const InetAddress &ip) { char ipbuf[INET6_ADDRSTRLEN]; Mutex::Lock _l(_ips_m); - lwip_init_interface((void*)this, this->_mac, ip); if (std::find(_ips.begin(),_ips.end(),ip) == _ips.end()) { + lwip_init_interface((void*)this, this->_mac, ip); + // TODO: Add ZTS_EVENT_ADDR_NEW ? _ips.push_back(ip); + // Send callback message + struct zts_addr_details *ad = new zts_addr_details; + ad->nwid = _nwid; + if (ip.isV4()) { + struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr); + memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4); + postEvent(ZTS_EVENT_ADDR_ADDED_IP4, (void*)ad); + } + if (ip.isV6()) { + struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr); + memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16); + postEvent(ZTS_EVENT_ADDR_ADDED_IP6, (void*)ad); + } std::sort(_ips.begin(),_ips.end()); } return true; @@ -155,15 +165,22 @@ bool VirtualTap::removeIp(const InetAddress &ip) { Mutex::Lock _l(_ips_m); std::vector::iterator i(std::find(_ips.begin(),_ips.end(),ip)); - //if (i == _ips.end()) { - // return false; - //} - _ips.erase(i); - if (ip.isV4()) { - // FIXME: De-register from network stack - } - if (ip.isV6()) { - // FIXME: De-register from network stack + if (std::find(_ips.begin(),_ips.end(),ip) != _ips.end()) { + struct zts_addr_details *ad = new zts_addr_details; + ad->nwid = _nwid; + if (ip.isV4()) { + struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr); + memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4); + postEvent(ZTS_EVENT_ADDR_REMOVED_IP4, (void*)ad); + // FIXME: De-register from network stack + } + if (ip.isV6()) { + // FIXME: De-register from network stack + struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr); + memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16); + postEvent(ZTS_EVENT_ADDR_REMOVED_IP6, (void*)ad); + } + _ips.erase(i); } return true; } diff --git a/src/VirtualTap.hpp b/src/VirtualTap.hpp index 03eda32..d7bf13c 100644 --- a/src/VirtualTap.hpp +++ b/src/VirtualTap.hpp @@ -81,6 +81,21 @@ public: void setEnabled(bool en); bool enabled() const; + /** + * Mutex for protecting IP address container for this tap. + */ + Mutex _ips_m; // Public because we want it accessible by the driver layer + + /** + * Return whether this tap has been assigned an IPv4 address. + */ + bool hasIpv4Addr(); + + /** + * Return whether this tap has been assigned an IPv6 address. + */ + bool hasIpv6Addr(); + /** * Adds an address to the userspace stack interface associated with this VirtualTap * - Starts VirtualTap main thread ONLY if successful @@ -158,23 +173,7 @@ public: // Lower-level lwIP netif handling and traffic handling readiness // ////////////////////////////////////////////////////////////////////////////// - void *netif4 = NULL; - void *netif6 = NULL; - - bool netif4WasUpLastCheck = false; - bool netif6WasUpLastCheck = false; - - /** - * Notes the current state of the lower level lwIP netif and reports if a state change - * has happened since the last check. This method is likely temporary. - */ - uint64_t recognizeLowerLevelInterfaceStateChange(void *n); - - /** - * A state will only be reported via callback if it differs from this value. Subsequently this - * value will be updated. - */ - //int _lastReportedStatus; + void *netif = NULL; /** * The last time that this virtual tap received a network config update from the core @@ -190,11 +189,6 @@ public: void lastConfigUpdate(uint64_t lastConfigUpdateTime); int _networkStatus = 0; - int _netifStatus = 0; - /** - * Returns whether or not this interface is ready for traffic. - */ - bool isReady(); ////////////////////////////////////////////////////////////////////////////// // Vars // @@ -203,9 +197,6 @@ public: std::vector > routes; char vtap_full_name[64]; - char vtap_abbr_name[16]; - - size_t ifindex = 0; std::vector ips() const; std::vector _ips; @@ -229,9 +220,6 @@ public: std::vector _multicastGroups; Mutex _multicastGroups_m; - Mutex _ips_m; - - //struct zts_network_details nd; /* * Timestamp of last run of housekeeping diff --git a/src/lwipDriver.cpp b/src/lwipDriver.cpp index 907ce34..4ff7d6a 100644 --- a/src/lwipDriver.cpp +++ b/src/lwipDriver.cpp @@ -54,7 +54,9 @@ #include "lwipDriver.hpp" #include "libzt.h" #include "Controls.hpp" -extern void postEvent(uint64_t id, int eventCode); + +extern void postEvent(uint64_t eventCode, void *arg); +extern void postEvent(uint64_t eventCode); #include "concurrentqueue.h" moodycamel::ConcurrentQueue rx_queue; @@ -100,7 +102,7 @@ static void tcpip_init_done(void *arg) sys_sem_t *sem; sem = (sys_sem_t *)arg; _run_lwip_tcpip = true; - postEvent((uint64_t)0, ZTS_EVENT_NETWORK_STACK_UP); + postEvent(ZTS_EVENT_STACK_UP); sys_sem_signal(sem); } @@ -113,7 +115,6 @@ void my_tcpip_callback(void *arg) int loop_score = LWIP_FRAMES_HANDLED_PER_CORE_CALL; // max num of packets to read per polling call struct zts_sorted_packet *sp; while (loop_score > 0 && rx_queue.size_approx() > 0) { - // TODO: Swap this block out for a thread-safe container struct pbuf *p; if (rx_queue.try_dequeue(sp)) { p = sp->p; @@ -155,6 +156,7 @@ static void main_lwip_driver_loop(void *arg) tcpip_callback_with_block(my_tcpip_callback, NULL, 1); } _has_exited = true; + postEvent(ZTS_EVENT_STACK_DOWN); } bool lwip_is_up() @@ -214,30 +216,23 @@ void lwip_driver_shutdown() void lwip_dispose_of_netifs(void *tapref) { VirtualTap *vtap = (VirtualTap*)tapref; - if (vtap->netif4) { - netif_remove((struct netif*)(vtap->netif4)); - netif_set_down((struct netif*)(vtap->netif4)); - netif_set_link_down((struct netif*)(vtap->netif4)); - delete vtap->netif4; - vtap->netif4 = NULL; - } - if (vtap->netif6) { - netif_remove((struct netif*)(vtap->netif6)); - netif_set_down((struct netif*)(vtap->netif6)); - netif_set_link_down((struct netif*)(vtap->netif6)); - delete vtap->netif6; - vtap->netif6 = NULL; + if (vtap->netif) { + netif_remove((struct netif*)(vtap->netif)); + netif_set_down((struct netif*)(vtap->netif)); + netif_set_link_down((struct netif*)(vtap->netif)); + delete vtap->netif; + vtap->netif = NULL; } } -err_t lwip_eth_tx(struct netif *netif, struct pbuf *p) +err_t lwip_eth_tx(struct netif *n, struct pbuf *p) { struct pbuf *q; char buf[ZT_MAX_MTU+32]; char *bufptr; int totalLength = 0; - VirtualTap *tap = (VirtualTap*)netif->state; + VirtualTap *tap = (VirtualTap*)n->state; bufptr = buf; for (q = p; q != NULL; q = q->next) { memcpy(bufptr, q->payload, q->len); @@ -256,7 +251,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p) int len = totalLength - sizeof(struct eth_hdr); int proto = Utils::ntoh((uint16_t)ethhdr->type); tap->_handler(tap->_arg, NULL, tap->_nwid, src_mac, dest_mac, proto, 0, data, len); - +/* if (ZT_MSG_TRANSFER == true) { char flagbuf[32]; memset(&flagbuf, 0, 32); @@ -270,7 +265,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p) DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(), Utils::ntoh(ethhdr->type), flagbuf); } - +*/ return ERR_OK; } @@ -285,7 +280,7 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e from.copyTo(ethhdr.src.addr, 6); to.copyTo(ethhdr.dest.addr, 6); ethhdr.type = Utils::hton((uint16_t)etherType); - +/* if (ZT_MSG_TRANSFER == true) { char flagbuf[32]; memset(&flagbuf, 0, 32); @@ -299,15 +294,9 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(), Utils::ntoh(ethhdr.type), flagbuf); } - - if (etherType == 0x0800 || etherType == 0x0806) { // ip4 or ARP - if (!tap->netif4) { - DEBUG_ERROR("dropped packet: no netif to accept this packet (etherType=%x) on this vtap (%p)", etherType, tap); - return; - } - } - if (etherType == 0x86DD) { // ip6 - if (!tap->netif6) { +*/ + if (etherType == 0x0800 || etherType == 0x0806 || etherType == 0x86DD) { // ip4 or ARP + if (!tap->netif) { DEBUG_ERROR("dropped packet: no netif to accept this packet (etherType=%x) on this vtap (%p)", etherType, tap); return; } @@ -354,10 +343,8 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e { case 0x0800: // ip4 case 0x0806: // ARP - sp->n = (struct netif *)tap->netif4; - break; case 0x86DD: // ip6 - sp->n = (struct netif *)tap->netif6; + sp->n = (struct netif *)tap->netif; break; default: DEBUG_ERROR("dropped packet: unhandled (etherType=%x)", etherType); @@ -366,29 +353,29 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e rx_queue.enqueue(sp); } -static void print_netif_info(struct netif *netif) { +static void print_netif_info(struct netif *n) { DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n", - netif, - netif->name[0], - netif->name[1], - netif->mtu, - netif->output, - netif->output_ip6, - netif->hwaddr[0], - netif->hwaddr[1], - netif->hwaddr[2], - netif->hwaddr[3], - netif->hwaddr[4], - netif->hwaddr[5], - netif->hwaddr_len, - netif->state, - netif->flags + n, + n->name[0], + n->name[1], + n->mtu, + n->output, + n->output_ip6, + n->hwaddr[0], + n->hwaddr[1], + n->hwaddr[2], + n->hwaddr[3], + n->hwaddr[4], + n->hwaddr[5], + n->hwaddr_len, + n->state, + n->flags ); } -bool lwip_is_netif_up(void *netif) +bool lwip_is_netif_up(void *n) { - return netif_is_up((struct netif*)netif); + return netif_is_up((struct netif*)n); } /** @@ -396,101 +383,134 @@ bool lwip_is_netif_up(void *netif) * - Interface is up/down (ZTS_EVENT_NETIF_UP, ZTS_EVENT_NETIF_DOWN) * - Address changes while up (ZTS_EVENT_NETIF_NEW_ADDRESS) */ -static void netif_status_callback(struct netif *netif) +static void netif_status_callback(struct netif *n) { + //DEBUG_INFO("netif=%p", n); // TODO: It appears that there may be a bug in lwIP's handling of callbacks for netifs // configured to handle ipv4 traffic. For this reason a temporary measure of checking // the status of the interfaces ourselves from the service is used. - if (!netif->state) { + /* + if (!n->state) { return; } - VirtualTap *tap = (VirtualTap *)netif->state; - if (netif->flags & NETIF_FLAG_UP) { - VirtualTap *vtap = (VirtualTap*)netif->state; - if (netif == vtap->netif6) { - // DEBUG_INFO("netif=%p, vtap->netif6=%p", netif, vtap->netif6); - postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP6); - } - if (netif == vtap->netif4) { - // DEBUG_INFO("netif=%p, vtap->netif4=%p", netif, vtap->netif4); - postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP4); + uint64_t mac = 0; + memcpy(&mac, n->hwaddr, n->hwaddr_len); + + VirtualTap *tap = (VirtualTap *)n->state; + if (n->flags & NETIF_FLAG_UP) { + VirtualTap *vtap = (VirtualTap*)n->state; + + if (n) { + struct zts_netif_details *ifd = new zts_netif_details; + ifd->nwid = tap->_nwid; + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_UP, (void*)ifd); } } - if (!(netif->flags & NETIF_FLAG_UP)) { - if (netif->flags & NETIF_FLAG_MLD6) { - postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP6); - } else { - postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP4); - } + if (!(n->flags & NETIF_FLAG_UP)) { + struct zts_netif_details *ifd = new zts_netif_details; + ifd->nwid = tap->_nwid; + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_DOWN, (void*)ifd); } + */ // TODO: ZTS_EVENT_NETIF_NEW_ADDRESS - //print_netif_info(netif); } /** * Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED) */ -static void netif_remove_callback(struct netif *netif) +static void netif_remove_callback(struct netif *n) { - if (!netif->state) { + if (!n->state) { return; } - VirtualTap *tap = (VirtualTap *)netif->state; - postEvent(tap->_nwid, ZTS_EVENT_NETIF_REMOVED); - //print_netif_info(netif); + VirtualTap *tap = (VirtualTap *)n->state; + uint64_t mac = 0; + memcpy(&mac, n->hwaddr, n->hwaddr_len); + struct zts_netif_details *ifd = new zts_netif_details; + ifd->nwid = tap->_nwid; + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_REMOVED, (void*)ifd); } /** * Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN) */ -static void netif_link_callback(struct netif *netif) +static void netif_link_callback(struct netif *n) { - if (!netif->state) { + if (!n->state) { return; } - VirtualTap *tap = (VirtualTap *)netif->state; - if (netif->flags & NETIF_FLAG_LINK_UP) { - postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_UP); + VirtualTap *tap = (VirtualTap *)n->state; + uint64_t mac = 0; + memcpy(&mac, n->hwaddr, n->hwaddr_len); + if (n->flags & NETIF_FLAG_LINK_UP) { + struct zts_netif_details *ifd = new zts_netif_details; + ifd->nwid = tap->_nwid; + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_LINK_UP, (void*)ifd); } - if (netif->flags & NETIF_FLAG_LINK_UP) { - postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_DOWN); + if (n->flags & NETIF_FLAG_LINK_UP) { + struct zts_netif_details *ifd = new zts_netif_details; + ifd->nwid = tap->_nwid; + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_LINK_DOWN, (void*)ifd); } - //print_netif_info(netif); } -static err_t netif_init_4(struct netif *netif) +void lwip_set_callbacks(struct netif *n) { - netif->hwaddr_len = 6; - netif->name[0] = 'z'; - netif->name[1] = '4'; - netif->linkoutput = lwip_eth_tx; - netif->output = etharp_output; - netif->mtu = ZT_MAX_MTU; - netif->flags = NETIF_FLAG_BROADCAST +#if LWIP_NETIF_STATUS_CALLBACK + netif_set_status_callback(n, netif_status_callback); +#endif +#if LWIP_NETIF_REMOVE_CALLBACK + netif_set_remove_callback(n, netif_remove_callback); +#endif +#if LWIP_NETIF_LINK_CALLBACK + netif_set_link_callback(n, netif_link_callback); +#endif +} + +static void lwip_prepare_netif_status_msg(struct netif *n) +{ + VirtualTap *tap = (VirtualTap*)(n->state); + struct zts_netif_details *ifd = new zts_netif_details; + // nwid + ifd->nwid = tap->_nwid; + // mtu + ifd->mtu = n->mtu; + // MAC + memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len); + ifd->mac = htonll(ifd->mac) >> 16; + postEvent(ZTS_EVENT_NETIF_UP, (void*)ifd); +} + +static err_t netif_init(struct netif *n) +{ + n->hwaddr_len = 6; + n->name[0] = 'z'; + n->name[1] = '4'; + n->linkoutput = lwip_eth_tx; + n->output = etharp_output; + n->mtu = ZT_MAX_MTU; + n->flags = NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_ETHERNET | NETIF_FLAG_IGMP + | NETIF_FLAG_MLD6 | NETIF_FLAG_LINK_UP | NETIF_FLAG_UP; - netif->hwaddr_len = sizeof(netif->hwaddr); - return ERR_OK; -} - -static err_t netif_init_6(struct netif *netif) -{ - netif->hwaddr_len = 6; - netif->name[0] = 'z'; - netif->name[1] = '6'; - netif->linkoutput = lwip_eth_tx; - netif->output = etharp_output; - netif->output_ip6 = ethip6_output; - netif->mtu = ZT_MAX_MTU; - netif->flags = NETIF_FLAG_BROADCAST - | NETIF_FLAG_ETHARP - | NETIF_FLAG_ETHERNET - | NETIF_FLAG_IGMP - | NETIF_FLAG_MLD6; - netif->hwaddr_len = sizeof(netif->hwaddr); + n->hwaddr_len = sizeof(n->hwaddr); + // lwip_set_callbacks(netif); + VirtualTap *tap = (VirtualTap*)(n->state); + tap->_mac.copyTo(n->hwaddr, n->hwaddr_len); + lwip_prepare_netif_status_msg(n); return ERR_OK; } @@ -498,7 +518,15 @@ void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip) { char ipbuf[INET6_ADDRSTRLEN]; char macbuf[ZTS_MAC_ADDRSTRLEN]; - struct netif *n = new struct netif; + + VirtualTap *vtap = (VirtualTap*)tapref; + struct netif *n = NULL; + if (vtap->netif) { + n = (struct netif*)vtap->netif; + } + else { + n = new struct netif; + } if (ip.isV4()) { char nmbuf[INET6_ADDRSTRLEN]; @@ -506,47 +534,34 @@ void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip) IP4_ADDR(&gw,127,0,0,1); ipaddr.addr = *((u32_t *)ip.rawIpData()); netmask.addr = *((u32_t *)ip.netmask().rawIpData()); - netif_add(n, &ipaddr, &netmask, &gw, NULL, netif_init_4, tcpip_input); - n->state = tapref; - mac.copyTo(n->hwaddr, n->hwaddr_len); + netif_add(n, &ipaddr, &netmask, &gw, tapref, netif_init, tcpip_input); +/* snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x", n->hwaddr[0], n->hwaddr[1], n->hwaddr[2], n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]); - DEBUG_INFO("initialized netif as [mac=%s, addr=%s, nm=%s]", + DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, nm=%s]",n, macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf)); - netif_set_up(n); - netif_set_link_up(n); - VirtualTap *vtap = (VirtualTap*)tapref; - vtap->netif4 = (void*)n; +*/ + vtap->netif = (void*)n; } - if (ip.isV6()) - { + if (ip.isV6()) { static ip6_addr_t ipaddr; memcpy(&(ipaddr.addr), ip.rawIpData(), sizeof(ipaddr.addr)); n->ip6_autoconfig_enabled = 1; - netif_add(n, NULL, NULL, NULL, NULL, netif_init_6, tcpip_input); + netif_ip6_addr_set(n, 1, &ipaddr); - n->state = tapref; - mac.copyTo(n->hwaddr, n->hwaddr_len); netif_create_ip6_linklocal_address(n, 1); netif_ip6_addr_set_state(n, 0, IP6_ADDR_TENTATIVE); netif_ip6_addr_set_state(n, 1, IP6_ADDR_TENTATIVE); - netif_set_default(n); - netif_set_up(n); - netif_set_link_up(n); + n->output_ip6 = ethip6_output; +/* snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x", n->hwaddr[0], n->hwaddr[1], n->hwaddr[2], n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]); - DEBUG_INFO("initialized netif as [mac=%s, addr=%s]", + DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s]", n, macbuf, ip.toString(ipbuf)); - VirtualTap *vtap = (VirtualTap*)tapref; - vtap->netif6 = (void*)n; +*/ } - // Set netif callbacks, these will be used to inform decisions made - // by the higher level callback monitor thread - netif_set_status_callback(n, netif_status_callback); - netif_set_remove_callback(n, netif_remove_callback); - netif_set_link_callback(n, netif_link_callback); } } // namespace ZeroTier \ No newline at end of file