Unfinished callback improvements, consolidated netif4 and netif6

Makefile

@@ -43,7 +43,7 @@ patch:
 
 .PHONY: clean
 clean:
-	rm -rf bin staging generated
+	rm -rf bin staging generated dist
 
 all: debug release
 

include/libzt.h

@@ -33,6 +33,8 @@
 #ifndef LIBZT_H
 #define LIBZT_H
 
+#include "lwipopts.h"
+
 #ifdef _WIN32
 	#ifdef ADD_EXPORTS
 		#define ZT_SOCKET_API __declspec(dllexport)
@@ -113,10 +115,9 @@ typedef int zts_err_t;
 #define ZTS_EVENT_NODE_ONLINE               2
 #define ZTS_EVENT_NODE_DOWN                 3
 #define ZTS_EVENT_NODE_IDENTITY_COLLISION   4
-// libzt node events
 #define ZTS_EVENT_NODE_UNRECOVERABLE_ERROR  16
 #define ZTS_EVENT_NODE_NORMAL_TERMINATION   17
-// Network-specific events
+// Network events
 #define ZTS_EVENT_NETWORK_NOT_FOUND         32
 #define ZTS_EVENT_NETWORK_CLIENT_TOO_OLD    33
 #define ZTS_EVENT_NETWORK_REQUESTING_CONFIG 34
@@ -125,23 +126,41 @@ typedef int zts_err_t;
 #define ZTS_EVENT_NETWORK_READY_IP4         37
 #define ZTS_EVENT_NETWORK_READY_IP6         38
 #define ZTS_EVENT_NETWORK_DOWN              39
-//
+// Network Stack events
-#define ZTS_EVENT_NETWORK_STACK_UP          48
+#define ZTS_EVENT_STACK_UP                  48
-#define ZTS_EVENT_NETWORK_STACK_DOWN        49
+#define ZTS_EVENT_STACK_DOWN                49
-
 // lwIP netif events
-#define ZTS_EVENT_NETIF_UP_IP4              64
+#define ZTS_EVENT_NETIF_UP                  64
-#define ZTS_EVENT_NETIF_UP_IP6              65
+#define ZTS_EVENT_NETIF_DOWN                65
-#define ZTS_EVENT_NETIF_DOWN_IP4            66
+#define ZTS_EVENT_NETIF_REMOVED             66
-#define ZTS_EVENT_NETIF_DOWN_IP6            67
+#define ZTS_EVENT_NETIF_LINK_UP             67
-#define ZTS_EVENT_NETIF_REMOVED             68
+#define ZTS_EVENT_NETIF_LINK_DOWN           68
-#define ZTS_EVENT_NETIF_LINK_UP             69
-#define ZTS_EVENT_NETIF_LINK_DOWN           70
-#define ZTS_EVENT_NETIF_NEW_ADDRESS         71
 // Peer events
 #define ZTS_EVENT_PEER_P2P                  96
 #define ZTS_EVENT_PEER_RELAY                97
 #define ZTS_EVENT_PEER_UNREACHABLE          98
+// Path events
+#define ZTS_EVENT_PATH_DISCOVERED           112
+#define ZTS_EVENT_PATH_ALIVE                113
+#define ZTS_EVENT_PATH_DEAD                 114
+// Route events
+#define ZTS_EVENT_ROUTE_ADDED               128
+#define ZTS_EVENT_ROUTE_REMOVED             129
+// Address events
+#define ZTS_EVENT_ADDR_ADDED_IP4            144
+#define ZTS_EVENT_ADDR_REMOVED_IP4          145
+#define ZTS_EVENT_ADDR_ADDED_IP6            146
+#define ZTS_EVENT_ADDR_REMOVED_IP6          147
+
+// Macros for legacy behaviour
+#define NODE_EVENT_TYPE(code) code >= ZTS_EVENT_NODE_UP && code <= ZTS_EVENT_NODE_NORMAL_TERMINATION
+#define NETWORK_EVENT_TYPE(code) code >= ZTS_EVENT_NETWORK_NOT_FOUND && code <= ZTS_EVENT_NETWORK_DOWN
+#define STACK_EVENT_TYPE(code) code >= ZTS_EVENT_STACK_UP && code <= ZTS_EVENT_STACK_DOWN
+#define NETIF_EVENT_TYPE(code) code >= ZTS_EVENT_NETIF_UP && code <= ZTS_EVENT_NETIF_LINK_DOWN
+#define PEER_EVENT_TYPE(code) code >= ZTS_EVENT_PEER_P2P && code <= ZTS_EVENT_PEER_UNREACHABLE
+#define PATH_EVENT_TYPE(code) code >= ZTS_EVENT_PATH_DISCOVERED && code <= ZTS_EVENT_PATH_DEAD
+#define ROUTE_EVENT_TYPE(code) code >= ZTS_EVENT_ROUTE_ADDED && code <= ZTS_EVENT_ROUTE_REMOVED
+#define ADDR_EVENT_TYPE(code) code >= ZTS_EVENT_ADDR_ADDED_IP4 && code <= ZTS_EVENT_ADDR_REMOVED_IP6
 
 //////////////////////////////////////////////////////////////////////////////
 // Common definitions and structures for interacting with the ZT socket API //
@@ -287,7 +306,7 @@ struct zts_in6_addr {
     u32_t u32_addr[4];
     u8_t  u8_addr[16];
   } un;
-#define s6_addr  un.u8_addr
+//#define s6_addr  un.u8_addr
 };
 
 struct zts_sockaddr_in {
@@ -416,6 +435,134 @@ enum zts_peer_role
 	ZTS_PEER_ROLE_PLANET = 2      // planetary root
 };
 
+/**
+ * A structure used to convey details about the current node
+ * to the user application
+ */
+struct zts_node_details
+{
+	/**
+	 * The node ID
+	 */
+	uint64_t address;
+
+	/**
+	 * The current clock value accord to the node
+	 */ 
+	uint64_t clock;
+
+	/**
+	 * Whether or not this node is online
+	 */
+	bool online;
+
+	/**
+	 * Whether port mapping is enabled
+	 */
+	bool portMappingEnabled;
+
+	/**
+	 * Whether multipath support is enabled. If true, this node will
+	 * be capable of utilizing multiple physical links simultaneosly
+	 * to create higher quality or more robust aggregate links.
+	 *
+	 * See: https://www.zerotier.com/manual.shtml#2_1_5
+	 */
+	bool multipathEnabled;
+
+	/**
+	 * The port used by the service to send and receive
+	 * all encapsulated traffic
+	 */
+	uint16_t primaryPort;
+
+	/**
+	 * Planet ID
+	 */
+	uint64_t planetWorldId;
+	uint64_t planetWorldTimestamp;
+	uint8_t versionMajor;
+	uint8_t versionMinor;
+	uint8_t versionRev;
+};
+
+/**
+ * A structure used to convey information to a user application via
+ * a callback function.
+ */
+struct zts_callback_msg
+{
+	zts_callback_msg():
+		eventCode(-1),
+		node(NULL),
+		network(NULL),
+		netif(NULL),
+		route(NULL),
+		path(NULL),
+		peer(NULL) {}
+
+	/**
+	 * Event identifier
+	 */
+	int eventCode;
+
+	struct zts_node_details *node;
+	struct zts_network_details *network;
+	struct zts_netif_details *netif;
+	struct zts_virtual_network_route *route;
+	struct zts_physical_path *path;
+	struct zts_peer_details *peer;
+	struct zts_addr_details *addr;
+};
+
+struct zts_addr_details
+{
+	uint64_t nwid;
+	struct sockaddr_storage addr;
+};
+
+/**
+ * A structure used to convey information about a virtual network
+ * interface (netif) to a user application.
+ */
+struct zts_netif_details
+{
+	/**
+	 * The virtual network that this interface was commissioned for.
+	 */
+	uint64_t nwid;
+
+	/**
+	 * The hardware address assigned to this interface
+	 */
+	uint64_t mac;
+
+	/**
+	 * The MTU for this interface
+	 */
+	int mtu;
+
+	/**
+	 * The IPv4 address assigned to this interface.
+	 */
+	//struct sockaddr_in ip4_addr;
+
+	/**
+	 * The IPv6 addresses assigned to this interface.
+	 */
+	//struct sockaddr_in6 ip6_addr[LWIP_IPV6_NUM_ADDRESSES];
+
+	/**
+	 * Number of IPv4 addresses assigned to this interface
+	 */
+	//int num_ip4_addr;
+
+	/**
+	 * Number of IPv6 addresses assigned to this interface
+	 */
+	//int num_ip6_addr;
+};
+
 /**
  * A structure used to represent a virtual network route
  */
@@ -590,7 +737,7 @@ extern "C" {
  * @param userCallbackFunc User-specified callback for ZeroTier events
  * @return 0 if successful; or 1 if failed
  */
-ZT_SOCKET_API int ZTCALL zts_start(const char *path, void (*userCallbackFunc)(uint64_t, int), int port = ZTS_DEFAULT_PORT);
+ZT_SOCKET_API int ZTCALL zts_start(const char *path, void (*userCallbackFunc)(struct zts_callback_msg*), int port = ZTS_DEFAULT_PORT);
 
 /**
  * @brief Stops the ZeroTier service, brings down all virtual interfaces in order to stop all traffic processing.

src/Controls.cpp

@@ -113,40 +113,81 @@ OneService *service;
 	static jmethodID _userCallbackMethodRef = NULL;
 #endif
 
-void (*_userEventCallbackFunc)(uint64_t, int);
+void (*_userEventCallbackFunc)(struct zts_callback_msg *);
 
-extern moodycamel::ConcurrentQueue<std::pair<uint64_t, int>*> _callbackMsgQueue;
+moodycamel::ConcurrentQueue<struct zts_callback_msg*> _callbackMsgQueue;
 
 //////////////////////////////////////////////////////////////////////////////
 // Internal ZeroTier Service Controls (user application shall not use these)//
 //////////////////////////////////////////////////////////////////////////////
 
-void postEvent(uint64_t id, int eventCode)
+void postEvent(int eventCode, void *arg)
 {
-    // Queue callback event messages from other threads (such as lwIP driver)
+	struct zts_callback_msg *msg = new zts_callback_msg();
-    _callbackMsgQueue.enqueue(new std::pair<uint64_t, int>(id, eventCode));
+	msg->eventCode = eventCode;
+
+	if (NODE_EVENT_TYPE(eventCode)) {
+		msg->node = (struct zts_node_details*)arg;
+	} if (NETWORK_EVENT_TYPE(eventCode)) {
+		msg->network = (struct zts_network_details*)arg;
+	} if (NETIF_EVENT_TYPE(eventCode)) {
+		msg->netif = (struct zts_netif_details*)arg;
+	} if (ROUTE_EVENT_TYPE(eventCode)) {
+		msg->route = (struct zts_virtual_network_route*)arg;
+	} if (PATH_EVENT_TYPE(eventCode)) {
+		msg->path = (struct zts_physical_path*)arg;
+	} if (PEER_EVENT_TYPE(eventCode)) {
+		msg->peer = (struct zts_peer_details*)arg;
+	} if (ADDR_EVENT_TYPE(eventCode)) {
+		msg->addr = (struct zts_addr_details*)arg;
+	}
+    _callbackMsgQueue.enqueue(msg);
 }
 
-void _process_callback_event_helper(uint64_t nwid, int eventCode)
+void postEvent(int eventCode) {
+	postEvent(eventCode, (void*)0);
+}
+
+void freeEvent(struct zts_callback_msg *msg)
+{
+	if (msg->node) { delete msg->node; }
+	if (msg->network) { delete msg->network; }
+	if (msg->netif) { delete msg->netif; }
+	if (msg->route) { delete msg->route; }
+	if (msg->path) { delete msg->path; }
+	if (msg->peer) { delete msg->peer; }
+	if (msg->addr) { delete msg->addr; }
+}
+
+void _process_callback_event_helper(struct zts_callback_msg *msg)
 {
 #ifdef SDK_JNI
 	if(_userCallbackMethodRef) {
 		JNIEnv *env;
 		jint rs = jvm->AttachCurrentThread(&env, NULL);
 		assert (rs == JNI_OK);
-		env->CallVoidMethod(objRef, _userCallbackMethodRef, nwid, eventCode);
+		if (NODE_EVENT_TYPE(msg->eventCode)) {
+			arg = msg->networkId;
+		}
+		if (NODE_EVENT_TYPE(msg->eventCode)) {
+			arg = msg->nodeId;
+		}
+		if (NODE_EVENT_TYPE(msg->eventCode)) {
+			arg = msg->nodeId;
+		}
+		env->CallVoidMethod(objRef, _userCallbackMethodRef, msg->networkId, msg->eventCode);
 	}
 #else
 	if (_userEventCallbackFunc) {
-		_userEventCallbackFunc(nwid, eventCode);
+		_userEventCallbackFunc(msg);
 	}
 #endif
 }
 
-void _process_callback_event(uint64_t nwid, int eventCode)
+void _process_callback_event(struct zts_callback_msg *msg)
 {
 	_callback_lock.lock();
-	_process_callback_event_helper(nwid, eventCode);
+	_process_callback_event_helper(msg);
 	_callback_lock.unlock();
 }
 
@@ -215,11 +256,11 @@ void *_zts_run_callbacks(void *thread_id)
 #endif
 	while (_run_callbacks || _callbackMsgQueue.size_approx() > 0)
     {
-        std::pair<uint64_t, int> *msg;
+        struct zts_callback_msg *msg;
-		for (int j = 0; j != 32; j++) { // TODO: Check size of queue
+		int sz = _callbackMsgQueue.size_approx();
+		for (int j = 0; j < sz; j++) {
 			if (_callbackMsgQueue.try_dequeue(msg)) {
-				// DEBUG_INFO("deqeueuing front: %llx,%d", msg->first, msg->second);
+				_process_callback_event(msg);
-                _process_callback_event(msg->first, msg->second);
 				delete msg;
 			}
 		}
@@ -272,12 +313,12 @@ void *_zts_run_service(void *arg)
 			switch(service->run()) {
 				case OneService::ONE_STILL_RUNNING:
 				case OneService::ONE_NORMAL_TERMINATION:
-					postEvent((uint64_t)0, ZTS_EVENT_NODE_NORMAL_TERMINATION);
+					postEvent(ZTS_EVENT_NODE_NORMAL_TERMINATION);
 					break;
 				case OneService::ONE_UNRECOVERABLE_ERROR:
 					DEBUG_ERROR("fatal error: %s", service->fatalErrorMessage().c_str());
 					err = true;
-					postEvent((uint64_t)0, ZTS_EVENT_NODE_UNRECOVERABLE_ERROR);
+					postEvent(ZTS_EVENT_NODE_UNRECOVERABLE_ERROR);
 					break;
 				case OneService::ONE_IDENTITY_COLLISION: {
 					err = true;
@@ -290,7 +331,7 @@ void *_zts_run_service(void *arg)
 						OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
 						OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
 					}
-					postEvent((uint64_t)0, ZTS_EVENT_NODE_IDENTITY_COLLISION);
+					postEvent(ZTS_EVENT_NODE_IDENTITY_COLLISION);
 				}	continue; // restart!
 			}
 			break; // terminate loop -- normally we don't keep restarting
@@ -300,7 +341,7 @@ void *_zts_run_service(void *arg)
 		delete service;
 		service = (OneService *)0;
 		_service_lock.unlock();
-		postEvent((uint64_t)0, ZTS_EVENT_NODE_DOWN);
+		postEvent(ZTS_EVENT_NODE_DOWN);
 	} catch ( ... ) {
 		DEBUG_ERROR("unexpected exception starting ZeroTier instance");
 	}
@@ -411,7 +452,7 @@ zts_err_t zts_deorbit(uint64_t moonWorldId)
 #ifdef SDK_JNI
 #endif
 
-zts_err_t zts_start(const char *path, void (*callback)(uint64_t, int), int port)
+zts_err_t zts_start(const char *path, void (*callback)(struct zts_callback_msg*), int port)
 {
 	Mutex::Lock _l(_service_lock);
 	lwip_driver_init();

src/Controls.hpp

@@ -39,7 +39,32 @@ namespace ZeroTier {
 // ZeroTier Internal Service Controls                                       //
 //////////////////////////////////////////////////////////////////////////////
 
-void postEvent(uint64_t id, int eventCode);
+/**
+ * Add a callback event message to the queue. This can be safely called
+ * from other threads since a lock-free queue is used.
+ *
+ * @param eventCode The event ID for this event
+ * @param msg Pointer to a structure of pointers to other message-relevant
+ * data structures.
+ */
+void postEvent(int eventCode, void *arg);
+
+/**
+ * Add a callback event message to the queue. This can be safely called
+ * from other threads since a lock-free queue is used. Note: For use in
+ * situations when no additional information needs to be conveyed to the
+ * user application.
+ *
+ * @param eventCode The event ID for this event
+ */
+void postEvent(int eventCode);
+
+/**
+ * Free whatever was allocated to contain the callback message
+ *
+ * @param msg Message to be freed
+ */
+void freeEvent(struct zts_callback_msg *msg);
 
 #ifdef __cplusplus
 extern "C" {

src/Options.h

@@ -5,11 +5,6 @@
 // Callbacks                                                                //
 //////////////////////////////////////////////////////////////////////////////
 
-#define ZTS_NODE_CALLBACKS    1
-#define ZTS_NETWORK_CALLBACKS 1
-#define ZTS_NETIF_CALLBACKS   1
-#define ZTS_PEER_CALLBACKS    1
-
 /**
  * The maximum number of un-processed callback messages
  */

src/Service.cpp

@@ -86,6 +86,9 @@
 #include <ifaddrs.h>
 #endif
 
+#include "libzt.h"
+#include "Controls.hpp"
+
 // Use the virtual netcon endpoint instead of a tun/tap port driver
 #include "VirtualTap.hpp"
 namespace ZeroTier { typedef VirtualTap EthernetTap; }
@@ -102,8 +105,7 @@ namespace ZeroTier { typedef VirtualTap EthernetTap; }
 
 namespace ZeroTier {
 
-// Concurrent queue for callback message processing
+extern void postEvent(uint64_t id, int eventCode);
-moodycamel::ConcurrentQueue<std::pair<uint64_t, int>*> _callbackMsgQueue;
 
 namespace {
 
@@ -803,7 +805,14 @@ public:
 	{
 		// Feed node events into lock-free queue for later dequeuing by the callback thread
 		if (event <= ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION) {
-			_callbackMsgQueue.enqueue(new std::pair<uint64_t, int>(0x0000000000, event));
+			if (event == ZTS_EVENT_NODE_ONLINE) {
+				struct zts_node_details *nd = new zts_node_details;
+				nd->address = _node->address();
+				postEvent(event, (void*)nd);
+			}
+			else {
+				postEvent(event, (void*)0);
+			}
 		}
 		switch(event) {
 			case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION: {
@@ -827,11 +836,12 @@ public:
 
 	inline void generateEventMsgs()
 	{
-		if (!lwip_is_up()) {
+		// Force the ordering of callback messages, these messages are
-			return; // Don't process peer status events unless the stack is up.
+		// only useful if the node and stack are both up and running
+		if (!_node->online() || !lwip_is_up()) {
+			return;
 		}
 		// Generate messages to be dequeued by the callback message thread
-#if ZTS_NETWORK_CALLBACKS
 		Mutex::Lock _l(_nets_m);
 		for(std::map<uint64_t,NetworkState>::iterator n(_nets.begin());n!=_nets.end();++n) {
 			int mostRecentStatus = n->second.config.status;
@@ -840,52 +850,63 @@ public:
 			if (n->second.tap->_networkStatus == mostRecentStatus) {
 				continue; // No state change
 			}
+			struct zts_network_details *nd = new zts_network_details;
+
+			//memcpy(nd, &(pl->peers[i]), sizeof(struct zts_network_details));
+			nd->nwid = nwid;
+
 			switch (mostRecentStatus) {
 				case ZT_NETWORK_STATUS_NOT_FOUND:
-					_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_NOT_FOUND));
+					postEvent(ZTS_EVENT_NETWORK_NOT_FOUND, (void*)nd);
 					break;
 				case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:
-					_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_CLIENT_TOO_OLD));
+					postEvent(ZTS_EVENT_NETWORK_CLIENT_TOO_OLD, (void*)nd);
 					break;
 				case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION:
-					_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_REQUESTING_CONFIG));
+					postEvent(ZTS_EVENT_NETWORK_REQUESTING_CONFIG, (void*)nd);
 					break;
 				case ZT_NETWORK_STATUS_OK:
-					if (tap->netif4 && lwip_is_netif_up(tap->netif4)) {
+					if (tap->hasIpv4Addr() && lwip_is_netif_up(tap->netif)) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_READY_IP4));
+						postEvent(ZTS_EVENT_NETWORK_READY_IP4, (void*)nd);
 					}
-					if (tap->netif6 && lwip_is_netif_up(tap->netif6)) {
+					if (tap->hasIpv6Addr() && lwip_is_netif_up(tap->netif)) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_READY_IP6));
+						postEvent(ZTS_EVENT_NETWORK_READY_IP6, (void*)nd);
 					}
-					_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_OK));
+					// In addition to the READY messages, send one OK message
+					postEvent(ZTS_EVENT_NETWORK_OK, (void*)nd);
 					break;
 				case ZT_NETWORK_STATUS_ACCESS_DENIED:
-					_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(nwid, ZTS_EVENT_NETWORK_ACCESS_DENIED));
+					postEvent(ZTS_EVENT_NETWORK_ACCESS_DENIED, (void*)nd);
 					break;
 				default:
 					break;
 			}
 			n->second.tap->_networkStatus = mostRecentStatus;
 		}
-#endif // ZTS_NETWORK_CALLBACKS
+
-#if ZTS_PEER_CALLBACKS
 		// TODO: Add ZTS_EVENT_PEER_NEW
 		ZT_PeerList *pl = _node->peers();
 		if (pl) {
 			for(unsigned long i=0;i<pl->peerCount;++i) {
+
+				struct zts_peer_details *pd = new zts_peer_details;
+
+				memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details));
+				// pl->peers[i].address, 0, 0, 0, NULL, 0);
+
 				if (!peerCache.count(pl->peers[i].address)) {
 					if (pl->peers[i].pathCount > 0) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(pl->peers[i].address, ZTS_EVENT_PEER_P2P));
+						postEvent(ZTS_EVENT_PEER_P2P, (void*)pd);
 					}
 					if (pl->peers[i].pathCount == 0) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(pl->peers[i].address, ZTS_EVENT_PEER_RELAY));
+						postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
 					}
 				} else {
 					if (peerCache[pl->peers[i].address] == 0 && pl->peers[i].pathCount > 0) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(pl->peers[i].address, ZTS_EVENT_PEER_P2P));
+						postEvent(ZTS_EVENT_PEER_P2P, (void*)pd);
 					}
 					if (peerCache[pl->peers[i].address] > 0 && pl->peers[i].pathCount == 0) {
-						_callbackMsgQueue.enqueue(new std::pair<uint64_t,int>(pl->peers[i].address, ZTS_EVENT_PEER_RELAY));
+						postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
 					}
 				}
 				// Update our cache with most recently observed path count
@@ -893,7 +914,6 @@ public:
 			}
 		}
 		_node->freeQueryResult((void *)pl);
-#endif // ZTS_PEER_CALLBACKS
 	}
 
 	inline int networkCount()

src/VirtualTap.cpp

@@ -48,7 +48,7 @@ namespace ZeroTier {
 
 class VirtualTap;
 extern OneService *service;
-extern void postEvent(uint64_t id, int eventCode);
+extern void postEvent(int eventCode, void *arg);
 
 /**
  * A virtual tap device. The ZeroTier core service creates one of these for each
@@ -86,7 +86,9 @@ VirtualTap::VirtualTap(
 
 VirtualTap::~VirtualTap()
 {
-	postEvent(_nwid, ZTS_EVENT_NETWORK_DOWN);
+	struct zts_network_details *nd = new zts_network_details;
+	nd->nwid = _nwid;
+	postEvent(ZTS_EVENT_NETWORK_DOWN, (void*)nd);
 	_run = false;
 	::write(_shutdownSignalPipe[1],"\0",1);
 	_phy.whack();
@@ -96,34 +98,6 @@ VirtualTap::~VirtualTap()
 	::close(_shutdownSignalPipe[1]);
 }
 
-uint64_t VirtualTap::recognizeLowerLevelInterfaceStateChange(void *n)
-{
-	if (!n) {
-		return ZTS_EVENT_NONE;
-	}
-	if (n == netif4) {
-		if (netif4WasUpLastCheck && !lwip_is_netif_up(netif4)) {
-			netif4WasUpLastCheck = false;
-			return ZTS_EVENT_NETIF_DOWN_IP4;
-		}
-		if (!netif4WasUpLastCheck && lwip_is_netif_up(netif4)) {
-			netif4WasUpLastCheck = true;
-			return ZTS_EVENT_NETIF_UP_IP4;
-		}
-	}
-	if (n == netif6) {
-		if (netif6WasUpLastCheck && !lwip_is_netif_up(netif6)) {
-			netif6WasUpLastCheck = false;
-			return ZTS_EVENT_NETIF_DOWN_IP6;
-		}
-		if (!netif6WasUpLastCheck && lwip_is_netif_up(netif6)) {
-			netif6WasUpLastCheck = true;
-			return ZTS_EVENT_NETIF_UP_IP6;
-		}
-	}
-	return ZTS_EVENT_NONE;
-}
-
 void VirtualTap::lastConfigUpdate(uint64_t lastConfigUpdateTime)
 {
 	_lastConfigUpdateTime = lastConfigUpdateTime;
@@ -139,13 +113,49 @@ bool VirtualTap::enabled() const
 	return _enabled;
 }
 
+bool VirtualTap::hasIpv4Addr()
+{
+	Mutex::Lock _l(_ips_m);
+	std::vector<InetAddress>::iterator it(_ips.begin());
+	while (it != _ips.end()) {
+		if ((*it).isV4()) { return true; }
+		it++;
+	}
+	return false;
+}
+
+bool VirtualTap::hasIpv6Addr()
+{
+	Mutex::Lock _l(_ips_m);
+	std::vector<InetAddress>::iterator it(_ips.begin());
+	while (it != _ips.end()) {
+		if ((*it).isV6()) { return true; }
+		it++;
+	}
+	return false;
+}
+
 bool VirtualTap::addIp(const InetAddress &ip)
 {
 	char ipbuf[INET6_ADDRSTRLEN];
 	Mutex::Lock _l(_ips_m);
-	lwip_init_interface((void*)this, this->_mac, ip);
 	if (std::find(_ips.begin(),_ips.end(),ip) == _ips.end()) {
+		lwip_init_interface((void*)this, this->_mac, ip);
+		// TODO: Add ZTS_EVENT_ADDR_NEW ?
 		_ips.push_back(ip);
+		// Send callback message
+		struct zts_addr_details *ad = new zts_addr_details;
+		ad->nwid = _nwid;
+		if (ip.isV4()) {
+			struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr);
+			memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4);
+			postEvent(ZTS_EVENT_ADDR_ADDED_IP4, (void*)ad);
+		}
+		if (ip.isV6()) {
+			struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr);
+			memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16);
+			postEvent(ZTS_EVENT_ADDR_ADDED_IP6, (void*)ad);
+		}
 		std::sort(_ips.begin(),_ips.end());
 	}
 	return true;
@@ -155,15 +165,22 @@ bool VirtualTap::removeIp(const InetAddress &ip)
 {
 	Mutex::Lock _l(_ips_m);
 	std::vector<InetAddress>::iterator i(std::find(_ips.begin(),_ips.end(),ip));
-	//if (i == _ips.end()) {
+	if (std::find(_ips.begin(),_ips.end(),ip) != _ips.end()) {
-	//	return false;
+		struct zts_addr_details *ad = new zts_addr_details;
-	//}
+		ad->nwid = _nwid;
-	_ips.erase(i);
+		if (ip.isV4()) {
-	if (ip.isV4()) {
+			struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr);
-		// FIXME: De-register from network stack
+			memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4);
-	}
+			postEvent(ZTS_EVENT_ADDR_REMOVED_IP4, (void*)ad);
-	if (ip.isV6()) {
+			// FIXME: De-register from network stack
-		// FIXME: De-register from network stack
+		}
+		if (ip.isV6()) {
+			// FIXME: De-register from network stack
+			struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr);
+			memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16);
+			postEvent(ZTS_EVENT_ADDR_REMOVED_IP6, (void*)ad);
+		}
+		_ips.erase(i);
 	}
 	return true;
 }

src/VirtualTap.hpp

@@ -81,6 +81,21 @@ public:
 	void setEnabled(bool en);
 	bool enabled() const;
 
+	/**
+	 * Mutex for protecting IP address container for this tap.
+	 */
+	Mutex _ips_m; // Public because we want it accessible by the driver layer
+
+	/**
+	 * Return whether this tap has been assigned an IPv4 address.
+	 */
+	bool hasIpv4Addr();
+
+	/**
+	 * Return whether this tap has been assigned an IPv6 address.
+	 */
+	bool hasIpv6Addr();
+
 	/**
 	 * Adds an address to the userspace stack interface associated with this VirtualTap
 	 * - Starts VirtualTap main thread ONLY if successful
@@ -158,23 +173,7 @@ public:
 	// Lower-level lwIP netif handling and traffic handling readiness           //
 	//////////////////////////////////////////////////////////////////////////////
 
-	void *netif4 = NULL;
+	void *netif = NULL;
-	void *netif6 = NULL;
-
-	bool netif4WasUpLastCheck = false;
-	bool netif6WasUpLastCheck = false;
-
-	/**
-	 * Notes the current state of the lower level lwIP netif and reports if a state change
-	 * has happened since the last check. This method is likely temporary.
-	 */
-	uint64_t recognizeLowerLevelInterfaceStateChange(void *n);
-
-	/**
-	 * A state will only be reported via callback if it differs from this value. Subsequently this
-	 * value will be updated.
-	 */
-	//int _lastReportedStatus;
 
 	/**
 	 * The last time that this virtual tap received a network config update from the core
@@ -190,11 +189,6 @@ public:
 	void lastConfigUpdate(uint64_t lastConfigUpdateTime);
 
 	int _networkStatus = 0;
-	int _netifStatus = 0;
-	/**
-	 * Returns whether or not this interface is ready for traffic.
-	 */
-	bool isReady();
 
 	//////////////////////////////////////////////////////////////////////////////
 	// Vars                                                                     //
@@ -203,9 +197,6 @@ public:
 	std::vector<std::pair<InetAddress, InetAddress> > routes;
 
 	char vtap_full_name[64];
-	char vtap_abbr_name[16];
-
-	size_t ifindex = 0;
 
 	std::vector<InetAddress> ips() const;
 	std::vector<InetAddress> _ips;
@@ -229,9 +220,6 @@ public:
 
 	std::vector<MulticastGroup> _multicastGroups;
 	Mutex _multicastGroups_m;
-	Mutex _ips_m;
-
-	//struct zts_network_details nd;
 
 	/*
 	 * Timestamp of last run of housekeeping

src/lwipDriver.cpp

@@ -54,7 +54,9 @@
 #include "lwipDriver.hpp"
 #include "libzt.h"
 #include "Controls.hpp"
-extern void postEvent(uint64_t id, int eventCode);
+
+extern void postEvent(uint64_t eventCode, void *arg);
+extern void postEvent(uint64_t eventCode);
 
 #include "concurrentqueue.h"
 moodycamel::ConcurrentQueue<struct ZeroTier::zts_sorted_packet*> rx_queue;
@@ -100,7 +102,7 @@ static void tcpip_init_done(void *arg)
 	sys_sem_t *sem;
 	sem = (sys_sem_t *)arg;
 	_run_lwip_tcpip = true;
-	postEvent((uint64_t)0, ZTS_EVENT_NETWORK_STACK_UP);
+	postEvent(ZTS_EVENT_STACK_UP);
 	sys_sem_signal(sem);
 }
 
@@ -113,7 +115,6 @@ void my_tcpip_callback(void *arg)
 	int loop_score = LWIP_FRAMES_HANDLED_PER_CORE_CALL; // max num of packets to read per polling call
 	struct zts_sorted_packet *sp;
 	while (loop_score > 0 && rx_queue.size_approx() > 0) {
-		// TODO: Swap this block out for a thread-safe container
 		struct pbuf *p;
 		if (rx_queue.try_dequeue(sp)) {
 			p = sp->p;
@@ -155,6 +156,7 @@ static void main_lwip_driver_loop(void *arg)
 		tcpip_callback_with_block(my_tcpip_callback, NULL, 1);
 	}
 	_has_exited = true;
+	postEvent(ZTS_EVENT_STACK_DOWN);
 }
 
 bool lwip_is_up()
@@ -214,30 +216,23 @@ void lwip_driver_shutdown()
 void lwip_dispose_of_netifs(void *tapref)
 {
 	VirtualTap *vtap = (VirtualTap*)tapref;
-	if (vtap->netif4) {
+	if (vtap->netif) {
-		netif_remove((struct netif*)(vtap->netif4));
+		netif_remove((struct netif*)(vtap->netif));
-		netif_set_down((struct netif*)(vtap->netif4));
+		netif_set_down((struct netif*)(vtap->netif));
-		netif_set_link_down((struct netif*)(vtap->netif4));
+		netif_set_link_down((struct netif*)(vtap->netif));
-		delete vtap->netif4;
+		delete vtap->netif;
-		vtap->netif4 = NULL;
+		vtap->netif = NULL;
-	}
-	if (vtap->netif6) {
-		netif_remove((struct netif*)(vtap->netif6));
-		netif_set_down((struct netif*)(vtap->netif6));
-		netif_set_link_down((struct netif*)(vtap->netif6));
-		delete vtap->netif6;
-		vtap->netif6 = NULL;
 	}
 }
 
-err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
+err_t lwip_eth_tx(struct netif *n, struct pbuf *p)
 {
 	struct pbuf *q;
 	char buf[ZT_MAX_MTU+32];
 	char *bufptr;
 	int totalLength = 0;
 
-	VirtualTap *tap = (VirtualTap*)netif->state;
+	VirtualTap *tap = (VirtualTap*)n->state;
 	bufptr = buf;
 	for (q = p; q != NULL; q = q->next) {
 		memcpy(bufptr, q->payload, q->len);
@@ -256,7 +251,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 	int len = totalLength - sizeof(struct eth_hdr);
 	int proto = Utils::ntoh((uint16_t)ethhdr->type);
 	tap->_handler(tap->_arg, NULL, tap->_nwid, src_mac, dest_mac, proto, 0, data, len);
-
+/*
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
@@ -270,7 +265,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
 			Utils::ntoh(ethhdr->type), flagbuf);
 	}
-
+*/
 	return ERR_OK;
 }
 
@@ -285,7 +280,7 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e
 	from.copyTo(ethhdr.src.addr, 6);
 	to.copyTo(ethhdr.dest.addr, 6);
 	ethhdr.type = Utils::hton((uint16_t)etherType);
-
+/*
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
@@ -299,15 +294,9 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e
 		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
 			Utils::ntoh(ethhdr.type), flagbuf);
 	}
-
+*/
-	if (etherType == 0x0800 || etherType == 0x0806) { // ip4 or ARP
+	if (etherType == 0x0800 || etherType == 0x0806 || etherType == 0x86DD) { // ip4 or ARP
-		if (!tap->netif4) {
+		if (!tap->netif) {
-			DEBUG_ERROR("dropped packet: no netif to accept this packet (etherType=%x) on this vtap (%p)", etherType, tap);
-			return;
-		}
-	}
-	if (etherType == 0x86DD) { // ip6
-		if (!tap->netif6) {
 			DEBUG_ERROR("dropped packet: no netif to accept this packet (etherType=%x) on this vtap (%p)", etherType, tap);
 			return;
 		}
@@ -354,10 +343,8 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e
 	{
 		case 0x0800: // ip4
 		case 0x0806: // ARP
-			sp->n = (struct netif *)tap->netif4;
-			break;
 		case 0x86DD: // ip6
-			sp->n = (struct netif *)tap->netif6;
+			sp->n = (struct netif *)tap->netif;
 			break;
 		default:
 			DEBUG_ERROR("dropped packet: unhandled (etherType=%x)", etherType);
@@ -366,29 +353,29 @@ void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int e
 	rx_queue.enqueue(sp);
 }
 
-static void print_netif_info(struct netif *netif) {
+static void print_netif_info(struct netif *n) {
 	DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n",
-		netif,
+		n,
-		netif->name[0],
+		n->name[0],
-		netif->name[1],
+		n->name[1],
-		netif->mtu,
+		n->mtu,
-		netif->output,
+		n->output,
-		netif->output_ip6,
+		n->output_ip6,
-		netif->hwaddr[0],
+		n->hwaddr[0],
-		netif->hwaddr[1],
+		n->hwaddr[1],
-		netif->hwaddr[2],
+		n->hwaddr[2],
-		netif->hwaddr[3],
+		n->hwaddr[3],
-		netif->hwaddr[4],
+		n->hwaddr[4],
-		netif->hwaddr[5],
+		n->hwaddr[5],
-		netif->hwaddr_len,
+		n->hwaddr_len,
-		netif->state,
+		n->state,
-		netif->flags
+		n->flags
 	);
 }
 
-bool lwip_is_netif_up(void *netif)
+bool lwip_is_netif_up(void *n)
 {
-	return netif_is_up((struct netif*)netif);
+	return netif_is_up((struct netif*)n);
 }
 
 /**
@@ -396,101 +383,134 @@ bool lwip_is_netif_up(void *netif)
  *  - Interface is up/down (ZTS_EVENT_NETIF_UP, ZTS_EVENT_NETIF_DOWN)
  *  - Address changes while up (ZTS_EVENT_NETIF_NEW_ADDRESS)
  */
-static void netif_status_callback(struct netif *netif)
+static void netif_status_callback(struct netif *n)
 {
+	//DEBUG_INFO("netif=%p", n);
 	// TODO: It appears that there may be a bug in lwIP's handling of callbacks for netifs
 	// configured to handle ipv4 traffic. For this reason a temporary measure of checking
 	// the status of the interfaces ourselves from the service is used.
-	if (!netif->state) {
+	/*
+	if (!n->state) {
 		return;
 	}
-	VirtualTap *tap = (VirtualTap *)netif->state;
+	uint64_t mac = 0;
-	if (netif->flags & NETIF_FLAG_UP) {
+	memcpy(&mac, n->hwaddr, n->hwaddr_len);
-		VirtualTap *vtap = (VirtualTap*)netif->state;
+
-		if (netif == vtap->netif6) {
+	VirtualTap *tap = (VirtualTap *)n->state;
-			// DEBUG_INFO("netif=%p, vtap->netif6=%p", netif, vtap->netif6);
+	if (n->flags & NETIF_FLAG_UP) {
-			postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP6);
+		VirtualTap *vtap = (VirtualTap*)n->state;
-		}
+
-		if (netif == vtap->netif4) {
+		if (n) {
-			// DEBUG_INFO("netif=%p, vtap->netif4=%p", netif, vtap->netif4);
+			struct zts_netif_details *ifd = new zts_netif_details;
-			postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP4);
+			ifd->nwid = tap->_nwid;
+			memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+			ifd->mac = htonll(ifd->mac) >> 16;
+			postEvent(ZTS_EVENT_NETIF_UP, (void*)ifd);
 		}
 	}
-	if (!(netif->flags & NETIF_FLAG_UP)) {
+	if (!(n->flags & NETIF_FLAG_UP)) {
-		if (netif->flags & NETIF_FLAG_MLD6) {
+		struct zts_netif_details *ifd = new zts_netif_details;
-			postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP6);
+		ifd->nwid = tap->_nwid;
-		} else {
+		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
-			postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP4);
+		ifd->mac = htonll(ifd->mac) >> 16;
-		}
+		postEvent(ZTS_EVENT_NETIF_DOWN, (void*)ifd);
 	}
+	*/
 	// TODO: ZTS_EVENT_NETIF_NEW_ADDRESS
-	//print_netif_info(netif);
 }
 
 /**
  * Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
  */
-static void netif_remove_callback(struct netif *netif)
+static void netif_remove_callback(struct netif *n)
 {
-	if (!netif->state) {
+	if (!n->state) {
 		return;
 	}
-	VirtualTap *tap = (VirtualTap *)netif->state;
+	VirtualTap *tap = (VirtualTap *)n->state;
-	postEvent(tap->_nwid, ZTS_EVENT_NETIF_REMOVED);
+	uint64_t mac = 0;
-	//print_netif_info(netif);
+	memcpy(&mac, n->hwaddr, n->hwaddr_len);
+	struct zts_netif_details *ifd = new zts_netif_details;
+	ifd->nwid = tap->_nwid;
+	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+	ifd->mac = htonll(ifd->mac) >> 16;
+	postEvent(ZTS_EVENT_NETIF_REMOVED, (void*)ifd);
 }
 
 /**
  * Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
  */
-static void netif_link_callback(struct netif *netif)
+static void netif_link_callback(struct netif *n)
 {
-	if (!netif->state) {
+	if (!n->state) {
 		return;
 	}
-	VirtualTap *tap = (VirtualTap *)netif->state;
+	VirtualTap *tap = (VirtualTap *)n->state;
-	if (netif->flags & NETIF_FLAG_LINK_UP) {
+	uint64_t mac = 0;
-		postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_UP);
+	memcpy(&mac, n->hwaddr, n->hwaddr_len);
+	if (n->flags & NETIF_FLAG_LINK_UP) {
+		struct zts_netif_details *ifd = new zts_netif_details;
+		ifd->nwid = tap->_nwid;
+		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+		ifd->mac = htonll(ifd->mac) >> 16;
+		postEvent(ZTS_EVENT_NETIF_LINK_UP, (void*)ifd);
 	}
-	if (netif->flags & NETIF_FLAG_LINK_UP) {
+	if (n->flags & NETIF_FLAG_LINK_UP) {
-		postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_DOWN);
+		struct zts_netif_details *ifd = new zts_netif_details;
+		ifd->nwid = tap->_nwid;
+		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+		ifd->mac = htonll(ifd->mac) >> 16;
+		postEvent(ZTS_EVENT_NETIF_LINK_DOWN, (void*)ifd);
 	}
-	//print_netif_info(netif);
 }
 
-static err_t netif_init_4(struct netif *netif)
+void lwip_set_callbacks(struct netif *n)
 {
-	netif->hwaddr_len = 6;
+#if LWIP_NETIF_STATUS_CALLBACK
-	netif->name[0]    = 'z';
+	netif_set_status_callback(n, netif_status_callback);
-	netif->name[1]    = '4';
+#endif
-	netif->linkoutput = lwip_eth_tx;
+#if LWIP_NETIF_REMOVE_CALLBACK
-	netif->output     = etharp_output;
+	netif_set_remove_callback(n, netif_remove_callback);
-	netif->mtu        = ZT_MAX_MTU;
+#endif
-	netif->flags      = NETIF_FLAG_BROADCAST
+#if LWIP_NETIF_LINK_CALLBACK
+	netif_set_link_callback(n, netif_link_callback);
+#endif
+}
+
+static void lwip_prepare_netif_status_msg(struct netif *n)
+{
+	VirtualTap *tap = (VirtualTap*)(n->state);
+	struct zts_netif_details *ifd = new zts_netif_details;
+	// nwid
+	ifd->nwid = tap->_nwid;
+	// mtu
+	ifd->mtu = n->mtu;
+	// MAC
+	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+	ifd->mac = htonll(ifd->mac) >> 16;
+	postEvent(ZTS_EVENT_NETIF_UP, (void*)ifd);
+}
+
+static err_t netif_init(struct netif *n)
+{
+	n->hwaddr_len = 6;
+	n->name[0]    = 'z';
+	n->name[1]    = '4';
+	n->linkoutput = lwip_eth_tx;
+	n->output     = etharp_output;
+	n->mtu        = ZT_MAX_MTU;
+	n->flags      = NETIF_FLAG_BROADCAST
 		| NETIF_FLAG_ETHARP
 		| NETIF_FLAG_ETHERNET
 		| NETIF_FLAG_IGMP
+		| NETIF_FLAG_MLD6
 		| NETIF_FLAG_LINK_UP
 		| NETIF_FLAG_UP;
-	netif->hwaddr_len = sizeof(netif->hwaddr);
+	n->hwaddr_len = sizeof(n->hwaddr);
-	return ERR_OK;
+	// lwip_set_callbacks(netif);
-}
+	VirtualTap *tap = (VirtualTap*)(n->state);
-
+	tap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
-static err_t netif_init_6(struct netif *netif)
+	lwip_prepare_netif_status_msg(n);
-{
-	netif->hwaddr_len = 6;
-	netif->name[0]    = 'z';
-	netif->name[1]    = '6';
-	netif->linkoutput = lwip_eth_tx;
-	netif->output     = etharp_output;
-	netif->output_ip6 = ethip6_output;
-	netif->mtu        = ZT_MAX_MTU;
-	netif->flags      = NETIF_FLAG_BROADCAST
-		| NETIF_FLAG_ETHARP
-		| NETIF_FLAG_ETHERNET
-		| NETIF_FLAG_IGMP
-		| NETIF_FLAG_MLD6;
-	netif->hwaddr_len = sizeof(netif->hwaddr);
 	return ERR_OK;
 }
 
@@ -498,7 +518,15 @@ void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip)
 {
 	char ipbuf[INET6_ADDRSTRLEN];
 	char macbuf[ZTS_MAC_ADDRSTRLEN];
-	struct netif *n = new struct netif;
+
+	VirtualTap *vtap = (VirtualTap*)tapref;
+	struct netif *n = NULL;
+	if (vtap->netif) {
+		n = (struct netif*)vtap->netif;
+	}
+	else {
+		n = new struct netif;
+	}
 
 	if (ip.isV4()) {
 		char nmbuf[INET6_ADDRSTRLEN];
@@ -506,47 +534,34 @@ void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip)
 		IP4_ADDR(&gw,127,0,0,1);
 		ipaddr.addr = *((u32_t *)ip.rawIpData());
 		netmask.addr = *((u32_t *)ip.netmask().rawIpData());
-		netif_add(n, &ipaddr, &netmask, &gw, NULL, netif_init_4, tcpip_input);
+		netif_add(n, &ipaddr, &netmask, &gw, tapref, netif_init, tcpip_input);
-		n->state = tapref;
+/*
-		mac.copyTo(n->hwaddr, n->hwaddr_len);
 		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
 			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
 			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
-		DEBUG_INFO("initialized netif as [mac=%s, addr=%s, nm=%s]", 
+		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, nm=%s]",n, 
 			macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf));
-		netif_set_up(n);
+*/
-		netif_set_link_up(n);
+		vtap->netif = (void*)n;
-		VirtualTap *vtap = (VirtualTap*)tapref;
-		vtap->netif4 = (void*)n;
 	}
-	if (ip.isV6())
+	if (ip.isV6()) {
-	{
 		static ip6_addr_t ipaddr;
 		memcpy(&(ipaddr.addr), ip.rawIpData(), sizeof(ipaddr.addr));
 		n->ip6_autoconfig_enabled = 1;
-		netif_add(n, NULL, NULL, NULL, NULL, netif_init_6, tcpip_input);
+
 		netif_ip6_addr_set(n, 1, &ipaddr);
-		n->state = tapref;
-		mac.copyTo(n->hwaddr, n->hwaddr_len);
 		netif_create_ip6_linklocal_address(n, 1);
 		netif_ip6_addr_set_state(n, 0, IP6_ADDR_TENTATIVE);
 		netif_ip6_addr_set_state(n, 1, IP6_ADDR_TENTATIVE);
-		netif_set_default(n);
+		n->output_ip6 = ethip6_output;
-		netif_set_up(n);
+/*
-		netif_set_link_up(n);
 		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
 			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
 			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
-		DEBUG_INFO("initialized netif as [mac=%s, addr=%s]", 
+		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s]", n,
 			macbuf, ip.toString(ipbuf));
-		VirtualTap *vtap = (VirtualTap*)tapref;
+*/
-		vtap->netif6 = (void*)n;
 	}
-	// Set netif callbacks, these will be used to inform decisions made
-	// by the higher level callback monitor thread
-	netif_set_status_callback(n, netif_status_callback);
-	netif_set_remove_callback(n, netif_remove_callback);
-	netif_set_link_callback(n, netif_link_callback);
 }
 
 } // namespace ZeroTier