Switch to MEM_LIBC_MALLOC usage in lwIP. Added event callbacks. Use of finer-grained locks in RX logic. CRCs disabled on inbound packets

CMakeLists.txt

@@ -193,7 +193,7 @@ set (ZTO_SRC_DIR "${PROJ_DIR}/ext/ZeroTierOne")
 set (LIBZT_SRC_DIR "${PROJ_DIR}/src")
 
 include_directories ("${LIBZT_SRC_DIR}")
-#include_directories ("${ZTO_SRC_DIR}/include")
+include_directories ("${ZTO_SRC_DIR}/include")
 include_directories ("${PROJ_DIR}")
 include_directories ("${ZTO_SRC_DIR}/osdep")
 include_directories ("${ZTO_SRC_DIR}/node")

Makefile

@@ -46,4 +46,5 @@ update:
 # Patch submodules
 patch:
 	-git -C ext/lwip apply ../lwip.patch
-	-git -C ext/lwip-contrib apply ../lwip-contrib.patch
+	-git -C ext/lwip-contrib apply ../lwip-contrib.patch
+	-git -C ext/ZeroTierOne apply ../ZeroTierOne.patch

ext/ZeroTierOne.patch

@@ -0,0 +1,17 @@
+diff --git a/service/OneService.cpp b/service/OneService.cpp
+index a1c53764..757863a8 100644
+--- a/service/OneService.cpp
++++ b/service/OneService.cpp
+@@ -2244,6 +2244,12 @@ public:
+ #endif
+ 					syncManagedStuff(n,true,true);
+ 					n.tap->setMtu(nwc->mtu);
++#if defined(ZT_SDK)
++					// Inform the virtual tap of the update
++					if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE) {
++						n.tap->lastConfigUpdate(OSUtils::now());
++					}
++#endif
+ 				} else {
+ 					_nets.erase(nwid);
+ 					return -999; // tap init failed

ext/lwip-contrib.patch

@@ -1,16 +1,33 @@
 diff --git a/ports/unix/port/include/arch/cc.h b/ports/unix/port/include/arch/cc.h
-index a3dac04..39fede7 100644
+index a3dac04..424285e 100644
 --- a/ports/unix/port/include/arch/cc.h
 +++ b/ports/unix/port/include/arch/cc.h
-@@ -34,7 +34,7 @@
+@@ -32,6 +32,8 @@
+ #ifndef LWIP_ARCH_CC_H
+ #define LWIP_ARCH_CC_H
  
++#include "include/Debug.hpp"
++
  /* see https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
  #if defined __ANDROID__
--#define LWIP_UNIX_ANDROID
+ #define LWIP_UNIX_ANDROID
-+//#define LWIP_UNIX_ANDROID
+@@ -55,7 +57,7 @@
- #elif defined __linux__
+ #endif
- #define LWIP_UNIX_LINUX
+ 
- #elif defined __APPLE__
+ #if defined(LWIP_UNIX_ANDROID) && defined(FD_SET)
+-typedef __kernel_fd_set fd_set;
++//typedef __kernel_fd_set fd_set;
+ #endif
+ 
+ struct sio_status_s;
+@@ -63,4 +65,7 @@ typedef struct sio_status_s sio_status_t;
+ #define sio_fd_t sio_status_t*
+ #define __sio_fd_t_defined
+ 
++// Comment out the following line to use lwIP's default diagnostic printing routine
++#define LWIP_PLATFORM_DIAG(x)    do {DEBUG_INFO x;} while(0)
++
+ #endif /* LWIP_ARCH_CC_H */
 diff --git a/ports/win32/include/arch/cc.h b/ports/win32/include/arch/cc.h
 index 334be07..9384b70 100644
 --- a/ports/win32/include/arch/cc.h

include/Constants.hpp

@@ -34,15 +34,31 @@
 #define LIBZT_CONSTANTS_HPP
 
 //////////////////////////////////////////////////////////////////////////////
-// Error codes returned by libzt API                                        //
+// Error codes returned by ZeroTier and the libzt API                       //
+// See ext/ZeroTierOne/include/ZeroTierOne.h                                //
 //////////////////////////////////////////////////////////////////////////////
 
 typedef int zts_err_t;
 
-#define ZTS_ERR_OK 0
+#define ZTS_ERR_OK                          0 // Everything is ok
-#define ZTS_ERR_INVALID_ARG -1 // A parameter provided by the user application is invalid (e.g. our of range, NULL, etc)
+#define ZTS_ERR_INVALID_ARG                -1 // A parameter provided by the user application is invalid (e.g. our of range, NULL, etc)
-#define ZTS_ERR_SERVICE -2 // The service isn't initialized or is for some other reason currently unavailable
+#define ZTS_ERR_SERVICE                    -2 // The service isn't initialized or is for some other reason currently unavailable
-#define ZTS_ERR_INVALID_OP -3 // For some reason this API operation is not permitted (perhaps the service is still starting?)
+#define ZTS_ERR_INVALID_OP                 -3 // For some reason this API operation is not permitted (perhaps the service is still starting?)
+
+#define ZTS_EVENT_NODE_ONLINE               0x01 // Node is online
+#define ZTS_EVENT_NODE_OFFLINE              0x02 // Node is offline
+#define ZTS_EVENT_NODE_DOWN                 0x03 // Node is shutting down
+#define ZTS_EVENT_NODE_IDENTITY_COLLISION   0x04 // Identity collision - check for duplicate instances
+#define ZTS_EVENT_NODE_UNRECOVERABLE_ERROR  0x05 // Something is seriously wrong
+#define ZTS_EVENT_NODE_NORMAL_TERMINATION   0x06 // Service thread has stopped
+
+#define ZTS_EVENT_NETWORK_NOT_FOUND         0x07
+#define ZTS_EVENT_NETWORK_CLIENT_TOO_OLD    0x08
+#define ZTS_EVENT_NETWORK_REQUESTING_CONFIG 0x09
+#define ZTS_EVENT_NETWORK_OK                0x0a
+#define ZTS_EVENT_NETWORK_ACCESS_DENIED     0x0b
+#define ZTS_EVENT_NETWORK_READY             0x0c
+#define ZTS_EVENT_NETWORK_DOWN              0x0d
 
 //////////////////////////////////////////////////////////////////////////////
 // libzt config                                                             //
@@ -113,6 +129,16 @@ typedef int zts_err_t;
  */
 #define ZTS_HOUSEKEEPING_INTERVAL 1000
 
+/**
+ * Name of the service thread (for debug purposes only)
+ */
+#define ZTS_SERVICE_THREAD_NAME "ZeroTierServiceThread"
+
+/**
+ * Name of the callback monitor thread (for debug purposes only)
+ */
+#define ZTS_EVENT_CALLBACK_THREAD_NAME "ZeroTierEventCallbackThread"
+
 //////////////////////////////////////////////////////////////////////////////
 // lwIP driver config                                                       //
 // For more LWIP configuration options see: include/lwipopts.h              //

include/ServiceControls.hpp

@@ -89,6 +89,22 @@ ZT_SOCKET_API int ZTCALL zts_get_service_port();
  */
 ZT_SOCKET_API int ZTCALL zts_start(const char *path, int blocking);
 
+/**
+ * @brief Starts the ZeroTier service and notifies user application of events via callback
+ *
+ * @usage Should be called at the beginning of your application. Will blocks until all of the following conditions are met:
+ * - ZeroTier core service has been initialized
+ * - Cryptographic identity has been generated or loaded from directory specified by `path`
+ * - Virtual network is successfully joined
+ * - IP address is assigned by network controller service
+ * @param path path directory where cryptographic identities and network configuration files are stored and retrieved
+ *              (`identity.public`, `identity.secret`)
+ * @param userCallbackFunc User-specified callback for ZeroTier events
+ * @param blocking whether or not this call will block until the entire service is up and running
+ * @return 0 if successful; or 1 if failed
+ */
+ZT_SOCKET_API int ZTCALL zts_start_with_callback(const char *path, void (*userCallbackFunc)(uint64_t, int), int blocking);
+
 /**
  * @brief Starts the ZeroTier service
  *

include/VirtualTap.hpp

@@ -77,6 +77,25 @@ public:
 
 	~VirtualTap();
 
+	/**
+	 * A state will only be reported via callback if it differs from this value. Subsequently this
+	 * value will be updated.
+	 */
+	int _lastReportedStatus;
+
+	/**
+	 * The last time that this virtual tap received a network config update from the core
+	 */
+	uint64_t _lastConfigUpdateTime = 0;
+
+	/**
+	 * The last time that a callback notification was sent to the user application signalling
+	 * that this interface is ready to process traffic.
+	 */
+	uint64_t _lastReadyReportTime = 0;
+	
+	void lastConfigUpdate(uint64_t lastConfigUpdateTime);
+
 	void setEnabled(bool en);
 	bool enabled() const;
 
@@ -163,7 +182,6 @@ public:
 	//////////////////////////////////////////////////////////////////////////////
 
 	std::vector<std::pair<InetAddress, InetAddress> > routes;
-	void *zt1ServiceRef = NULL;
 
 	char vtap_full_name[64];
 	char vtap_abbr_name[16];

include/VirtualTapManager.hpp

@@ -30,14 +30,14 @@
  * Management of virtual tap interfaces
  */
 
+#ifndef LIBZT_VIRTUAL_TAP_MANAGER_H
+#define LIBZT_VIRTUAL_TAP_MANAGER_H
+
 #include "VirtualTap.hpp"
 #include "OneService.hpp"
 
 namespace ZeroTier {
 
-extern std::vector<void*> vtaps;
-extern Mutex _vtaps_lock;
-
 class VirtualTap;
 
 /**
@@ -47,94 +47,16 @@ class VirtualTapManager
 {
 public:
 
-	static void add_tap(VirtualTap *tap) {
+	static void add_tap(VirtualTap *tap);
-		_vtaps_lock.lock();
+	static VirtualTap *getTapByNWID(uint64_t nwid);
-		vtaps.push_back((void*)tap);
+	static size_t get_vtaps_size();
-		_vtaps_lock.unlock();
+	static void remove_by_nwid(uint64_t nwid);
-	}
+	static void clear();
-
+	static void get_network_details_helper(void *zt1ServiceRef, uint64_t nwid, struct zts_network_details *nd);
-	static VirtualTap *getTapByNWID(uint64_t nwid)
+	static void get_network_details(void *zt1ServiceRef, uint64_t nwid, struct zts_network_details *nd);
-	{
+	static void get_all_network_details(void *zt1ServiceRef, struct zts_network_details *nds, int *num);
-		_vtaps_lock.lock();
-		VirtualTap *s, *tap = nullptr;
-		for (size_t i=0; i<vtaps.size(); i++) {
-			s = (VirtualTap*)vtaps[i];
-			if (s->_nwid == nwid) { tap = s; }
-		}
-		_vtaps_lock.unlock();
-		return tap;
-	}
-
-	static size_t get_vtaps_size() {
-		size_t sz;
-		_vtaps_lock.lock();
-		sz = vtaps.size();
-		_vtaps_lock.unlock();
-		return sz;
-	}
-
-	// TODO: We shouldn't re-apply the reference to everything all the time
-	static void update_service_references(void *serviceRef) {
-		_vtaps_lock.lock();
-		for (size_t i=0;i<vtaps.size(); i++) {
-			VirtualTap *s = (VirtualTap*)vtaps[i];
-			s->zt1ServiceRef=serviceRef;
-		}
-		_vtaps_lock.unlock();
-	}
-
-	static void remove_by_nwid(uint64_t nwid) {
-		_vtaps_lock.lock();
-		for (size_t i=0;i<vtaps.size(); i++) {
-			VirtualTap *s = (VirtualTap*)vtaps[i];
-			if (s->_nwid == nwid) {
-				vtaps.erase(vtaps.begin() + i);
-			}
-		}
-		_vtaps_lock.unlock();
-	}
-
-	static void clear() {
-		_vtaps_lock.lock();
-		vtaps.clear();
-		_vtaps_lock.unlock();
-	}
-
-	static void get_network_details(uint64_t nwid, struct zts_network_details *nd)
-	{
-		VirtualTap *tap;
-		socklen_t addrlen;
-		_vtaps_lock.lock();
-		for (size_t i=0; i<vtaps.size(); i++) {
-			tap = (VirtualTap*)vtaps[i];
-			if (tap->_nwid == nwid) {
-				nd->nwid = tap->_nwid;
-				nd->mtu = tap->_mtu;
-				// assigned addresses
-				nd->num_addresses = tap->_ips.size() < ZTS_MAX_ASSIGNED_ADDRESSES ? tap->_ips.size() : ZTS_MAX_ASSIGNED_ADDRESSES;
-				for (int j=0; j<nd->num_addresses; j++) {
-					addrlen = tap->_ips[j].isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-					memcpy(&(nd->addr[j]), &(tap->_ips[j]), addrlen);
-				}
-				// routes
-				nd->num_routes = ZTS_MAX_NETWORK_ROUTES;
-				OneService *zt1Service = (OneService*)tap->zt1ServiceRef;
-				zt1Service->getRoutes(nwid, (ZT_VirtualNetworkRoute*)&(nd->routes)[0], &(nd->num_routes));
-				break;
-			}
-		}
-		_vtaps_lock.unlock();
-	}
-
-	static void get_all_network_details(struct zts_network_details *nds, int *num)
-	{
-		VirtualTap *tap;
-		*num = get_vtaps_size();
-		for (size_t i=0; i<vtaps.size(); i++) {
-			tap = (VirtualTap*)vtaps[i];
-			get_network_details(tap->_nwid, &nds[i]);
-		}
-	}
 };
 
-} // namespace ZeroTier
+} // namespace ZeroTier
+
+#endif // _H

include/libzt.h

@@ -100,7 +100,7 @@ typedef SSIZE_T ssize_t;
 #define ZTS_PF_INET         ZTS_AF_INET
 #define ZTS_PF_INET6        ZTS_AF_INET6
 #define ZTS_PF_UNSPEC       ZTS_AF_UNSPEC
-// 
+// Protocol command types
 #define ZTS_IPPROTO_IP      0x0000
 #define ZTS_IPPROTO_ICMP    0x0001
 #define ZTS_IPPROTO_TCP     0x0006
@@ -121,7 +121,7 @@ typedef SSIZE_T ssize_t;
 // fnctl() flags
 #define ZTS_O_NONBLOCK      0x0001
 #define ZTS_O_NDELAY        0x0001
-//
+// Shutdown commands
 #define ZTS_SHUT_RD         0x0000
 #define ZTS_SHUT_WR         0x0001
 #define ZTS_SHUT_RDWR       0x0002
@@ -161,7 +161,7 @@ typedef SSIZE_T ssize_t;
 // IPPROTO_IPV6 options
 #define ZTS_IPV6_CHECKSUM   0x0007  // RFC3542
 #define ZTS_IPV6_V6ONLY     0x001b  // RFC3493
-//
+// Macro's for defining ioctl() command values
 #define ZTS_IOCPARM_MASK    0x7fU
 #define ZTS_IOC_VOID        0x20000000UL
 #define ZTS_IOC_OUT         0x40000000UL
@@ -170,11 +170,10 @@ typedef SSIZE_T ssize_t;
 #define ZTS_IO(x,y)         (ZTS_IOC_VOID | ((x)<<8)|(y))
 #define ZTS_IOR(x,y,t)      (ZTS_IOC_OUT  | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
 #define ZTS_IOW(x,y,t)      (ZTS_IOC_IN   | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
-//
+// ioctl() commands
 #define ZTS_FIONREAD        ZTS_IOR('f', 127, unsigned long)
 #define ZTS_FIONBIO         ZTS_IOW('f', 126, unsigned long)
 
-
 /* FD_SET used for lwip_select */
 
 #ifndef ZTS_FD_SET
@@ -330,6 +329,9 @@ struct sockaddr_ll {
 extern "C" {
 #endif
 
+// Custom errno to prevent conflicts with platform's own errno
+extern int zts_errno;
+
 /**
  * @brief Create a socket
  *

src/ServiceControls.cpp

@@ -30,16 +30,11 @@
  * ZeroTier service controls
  */
 
-
+#include <queue>
 
 #include "OneService.hpp"
 #include "Node.hpp"
-
+#include "ZeroTierOne.h"
-namespace ZeroTier
-{
-	std::vector<void*> vtaps;
-	Mutex _vtaps_lock;
-}
 
 #include "Constants.hpp"
 #include "VirtualTapManager.hpp"
@@ -47,27 +42,64 @@ namespace ZeroTier
 #include "OSUtils.hpp"
 #include "ServiceControls.hpp"
 
-//#define SDK_JNI 1
+#include "lwip/stats.h"
-#ifdef SDK_JNI
-#include <jni.h>
-#endif
-
-ZeroTier::Mutex _service_lock;
-ZeroTier::Mutex _startup_lock;
-
-static ZeroTier::OneService *zt1Service;
-std::string homeDir;
-int servicePort = ZTS_DEFAULT_PORT;
-bool _freeHasBeenCalled = false;
-bool _serviceIsShuttingDown = false;
-bool _startupError = false;
 
 #if defined(_WIN32)
 WSADATA wsaData;
 #include <Windows.h>
 #endif
 
+#ifdef SDK_JNI
+#include <jni.h>
+#endif
+
+namespace ZeroTier {
+
+/*
+ * A lock used to protect any call which relies on the presence of a valid pointer
+ * to the ZeroTier service.
+ */
+Mutex _service_lock;
+
+/*
+ * A lock which protects flags and state variables used during the startup and
+ * shutdown phase.
+ */
+Mutex _startup_lock;
+
+/*
+ * A lock used to protect callback method pointers. With a coarser-grained lock it
+ * would be possible for one thread to alter the callback method pointer causing 
+ * undefined behaviour.
+ */
+Mutex _callback_lock;
+
+std::string homeDir;
+int servicePort = ZTS_DEFAULT_PORT;
+bool _freeHasBeenCalled = false;
+bool _serviceIsShuttingDown = false;
+bool _startupError = false;
+bool _nodeIsOnlineToggle = false;
+
 pthread_t service_thread;
+pthread_t callback_thread;
+
+// Collection of virtual tap interfaces
+std::map<uint64_t, VirtualTap*> vtapMap;
+Mutex _vtaps_lock;
+
+// Global reference to ZeroTier service
+OneService *zt1Service;
+
+// User-provided callback for ZeroTier events
+void (*_userCallbackFunc)(uint64_t, int);
+#ifdef SDK_JNI
+// Global references to JNI objects and VM kept for future callbacks
+static JavaVM *jvm = NULL;
+jobject objRef = NULL;
+jmethodID _userCallbackMethodRef = NULL;
+#endif
+}
 
 using namespace ZeroTier;
 
@@ -75,7 +107,69 @@ using namespace ZeroTier;
 // Internal ZeroTier Service Controls (user application shall not use these)//
 //////////////////////////////////////////////////////////////////////////////
 
-void api_sleep(int interval_ms)
+std::queue<std::pair<uint64_t, int>*> _callbackMsgQueue;
+
+void _push_callback_event(uint64_t nwid, int eventCode)
+{
+	_callback_lock.lock();
+	if (_callbackMsgQueue.size() >= 128) {
+		DEBUG_ERROR("too many callback messages in queue");
+		_callback_lock.unlock();
+		return;
+	}
+	_callbackMsgQueue.push(new std::pair<uint64_t, int>(nwid,eventCode));
+	_callback_lock.unlock();
+}
+
+void _process_callback_event_helper(uint64_t nwid, int eventCode)
+{
+#ifdef SDK_JNI
+	if(_userCallbackMethodRef) {
+		JNIEnv *env;
+		jint rs = jvm->AttachCurrentThread(&env, NULL);
+		assert (rs == JNI_OK);
+		env->CallVoidMethod(objRef, _userCallbackMethodRef, nwid, eventCode);
+	}
+#else
+	if (_userCallbackFunc) {
+		_userCallbackFunc((uint64_t)0, ZTS_EVENT_NODE_OFFLINE);
+	}
+#endif
+}
+
+void _process_callback_event(uint64_t nwid, int eventCode)
+{
+	_callback_lock.lock();
+	_process_callback_event_helper(nwid, eventCode);
+	_callback_lock.unlock();
+}
+
+bool _is_callback_registered()
+{
+	_callback_lock.lock();
+	bool retval = false;
+#ifdef SDK_JNI
+	retval = (jvm && objRef && _userCallbackMethodRef);
+#else
+	retval = _userCallbackFunc;
+#endif
+	_callback_lock.unlock();
+	return retval;
+}
+
+void _clear_registered_callback()
+{
+	_callback_lock.lock();
+#ifdef SDK_JNI
+	objRef = NULL;
+	_userCallbackMethodRef = NULL;
+#else
+	_userCallbackFunc = NULL;
+#endif
+	_callback_lock.unlock();
+}
+
+void _api_sleep(int interval_ms)
 {
 #if defined(_WIN32)
 	Sleep(interval_ms);
@@ -107,6 +201,90 @@ void _hibernate_if_needed()
 #ifdef SDK_JNI
 #endif
 
+/*
+ * Monitors the conditions required for triggering callbacks into user code. This was made
+ * into its own thread to prevent user application abuse of callbacks from affecting
+ * the timing of more sensitive aspects of the library such as polling and RX/TX of packets
+ */
+#if defined(_WIN32)
+DWORD WINAPI _zts_monitor_callback_conditions(LPVOID thread_id)
+#else
+void *_zts_monitor_callback_conditions(void *thread_id)
+#endif
+{
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_EVENT_CALLBACK_THREAD_NAME);
+#endif
+	while (!_serviceIsShuttingDown)
+	{
+		if (!_zts_node_online()) {
+			if (_nodeIsOnlineToggle) {
+				_nodeIsOnlineToggle = false;
+				_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_OFFLINE);
+			}
+		} if (_zts_node_online()) { // Only process pending network callbacks if the node is online
+			if (!_nodeIsOnlineToggle) {
+				_nodeIsOnlineToggle = true;
+				_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_ONLINE);
+			}
+
+			// First, handle queued messages from other threads
+			_callback_lock.lock();
+			while (_callbackMsgQueue.size()) {
+				std::pair<uint64_t,int> *msg = _callbackMsgQueue.front();
+				_callbackMsgQueue.pop();
+				_process_callback_event_helper(msg->first, msg->second);
+				delete msg;
+			}
+			_callback_lock.unlock();
+
+			// Second, inspect network states for changes we should report
+			_vtaps_lock.lock();
+			ZT_VirtualNetworkList *nl = zt1Service->getNode()->networks();
+			for(unsigned long i=0;i<nl->networkCount;++i) {
+				OneService::NetworkSettings localSettings;
+				zt1Service->getNetworkSettings(nl->networks[i].nwid,localSettings);
+				if (vtapMap[nl->networks[i].nwid]->_lastReportedStatus != nl->networks[i].status) {
+					switch (nl->networks[i].status) {
+						case ZT_NETWORK_STATUS_NOT_FOUND:
+							_process_callback_event(nl->networks[i].nwid, ZTS_EVENT_NETWORK_NOT_FOUND);
+							break;
+						case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:
+							_process_callback_event(nl->networks[i].nwid, ZTS_EVENT_NETWORK_CLIENT_TOO_OLD);
+							break;
+						case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION:
+							_process_callback_event(nl->networks[i].nwid, ZTS_EVENT_NETWORK_REQUESTING_CONFIG);
+							break;
+						case ZT_NETWORK_STATUS_OK:
+							_process_callback_event(nl->networks[i].nwid, ZTS_EVENT_NETWORK_OK);
+							break;
+						case ZT_NETWORK_STATUS_ACCESS_DENIED:
+							_process_callback_event(nl->networks[i].nwid, ZTS_EVENT_NETWORK_ACCESS_DENIED);
+							break;
+						default:
+							break;
+					}
+					vtapMap[nl->networks[i].nwid]->_lastReportedStatus = nl->networks[i].status;
+				}
+			}
+			zt1Service->getNode()->freeQueryResult((void *)nl);
+			// Finally, check for a more useful definition of "readiness"
+			std::map<uint64_t, VirtualTap*>::iterator it;
+			for (it = vtapMap.begin(); it != vtapMap.end(); it++) {
+				VirtualTap *tap = it->second;
+				if (tap->_lastConfigUpdateTime > 0 && !tap->_lastReadyReportTime && tap->_ips.size() > 0) {
+					tap->_lastReadyReportTime = tap->_lastConfigUpdateTime;
+					_process_callback_event(tap->_nwid, ZTS_EVENT_NETWORK_READY);
+				}
+			}
+    		_vtaps_lock.unlock();
+		}
+		// Doesn't need to happen as often as other API operations
+		_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+	}
+	DEBUG_ERROR("exiting from monitor loop");
+}
+
 // Starts a ZeroTier service in the background
 #if defined(_WIN32)
 DWORD WINAPI _zts_start_service(LPVOID thread_id)
@@ -114,8 +292,10 @@ DWORD WINAPI _zts_start_service(LPVOID thread_id)
 void *_zts_start_service(void *thread_id)
 #endif
 {	
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_SERVICE_THREAD_NAME);
+#endif
 	void *retval;
-	//DEBUG_INFO("identities are stored in path (%s)", homeDir.c_str());
 	zt1Service = (OneService *)0;
 
 	if (!homeDir.length()) {
@@ -155,10 +335,12 @@ void *_zts_start_service(void *thread_id)
 				switch(zt1Service->run()) {
 					case OneService::ONE_STILL_RUNNING:
 					case OneService::ONE_NORMAL_TERMINATION:
+						_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_NORMAL_TERMINATION);
 						break;
 					case OneService::ONE_UNRECOVERABLE_ERROR:
 						DEBUG_ERROR("fatal error: %s", zt1Service->fatalErrorMessage().c_str());
 						_startupError = true;
+						_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_UNRECOVERABLE_ERROR);
 						break;
 					case OneService::ONE_IDENTITY_COLLISION: {
 						_startupError = true;
@@ -171,6 +353,7 @@ void *_zts_start_service(void *thread_id)
 							OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
 							OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
 						}
+						_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_IDENTITY_COLLISION);
 					}	continue; // restart!
 				}
 				break; // terminate loop -- normally we don't keep restarting
@@ -183,6 +366,7 @@ void *_zts_start_service(void *thread_id)
 		zt1Service = (OneService *)0;
 		_service_lock.unlock();
 		_serviceIsShuttingDown = false;
+		_process_callback_event((uint64_t)0, ZTS_EVENT_NODE_DOWN);
 	} catch ( ... ) {
 		DEBUG_ERROR("unexpected exception starting ZeroTier instance");
 	}
@@ -197,6 +381,18 @@ extern "C" {
 // ZeroTier Service Controls                                                //
 //////////////////////////////////////////////////////////////////////////////
 
+#ifdef SDK_JNI
+/*
+ * Called from Java, saves a reference to the VM so it can be used later to call
+ * a user-specified callback method from C.
+ */
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_init(JNIEnv *env, jobject thisObj)
+{
+	jint rs = env->GetJavaVM(&jvm);
+   assert (rs == JNI_OK);
+}
+#endif
+
 zts_err_t zts_set_service_port(int portno)
 {
 	zts_err_t retval = ZTS_ERR_OK;
@@ -234,107 +430,6 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_get_1service_1port(
 	return zts_get_service_port();
 }
 #endif
-/*
-int zts_get_address(const uint64_t nwid, struct sockaddr_storage *addr, 
-	const int address_family)
-{
-	int err = -1;
-	if (!zt1Service) {
-		return ZTS_ERR_SERVICE;
-	}
-	VirtualTap *tap = getTapByNWID(nwid);
-	if (!tap) {
-		return -1;
-	}
-	_vtaps_lock.lock();
-	socklen_t addrlen = address_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-	for (size_t i=0; i<tap->_ips.size(); i++) {
-		if (address_family == AF_INET) {
-			if (tap->_ips[i].isV4()) {
-				memcpy(addr, &(tap->_ips[i]), addrlen);
-				addr->ss_family = AF_INET;
-				err = 0;
-				break;
-			}
-		}
-		if (address_family == AF_INET6) {
-			if (tap->_ips[i].isV6()) {
-				memcpy(addr, &(tap->_ips[i]), addrlen);
-				addr->ss_family = AF_INET6;
-				err = 0;
-				break;
-			}
-		}
-	}
-	_vtaps_lock.unlock();
-	return err; // nothing found
-}
-#ifdef SDK_JNI
-JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_get_1address(
-	JNIEnv *env, jobject thisObj, jlong nwid, jint address_family, jobject addr)
-{
-	struct sockaddr_storage ss;
-	int err = zts_get_address((uint64_t)nwid, &ss, address_family);
-	ss2zta(env, &ss, addr);
-	return err;
-}
-#endif
-
-int zts_has_address(const uint64_t nwid)
-{
-	struct sockaddr_storage ss;
-	memset(&ss, 0, sizeof(ss));
-	zts_get_address(nwid, &ss, AF_INET);
-	if (ss.ss_family == AF_INET) {
-		return true;
-	}
-	zts_get_address(nwid, &ss, AF_INET6);
-	if (ss.ss_family == AF_INET6) {
-		return true;
-	}
-	return false;
-}
-#ifdef SDK_JNI
-JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_has_1address(
-	JNIEnv *env, jobject thisObj, jlong nwid)
-{
-	return zts_has_address(nwid);
-}
-#endif
-*/
-
-/*
-void zts_get_6plane_addr(struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId)
-{
-	ZeroTier::InetAddress _6planeAddr = ZeroTier::InetAddress::makeIpv66plane(nwid,nodeId);
-	memcpy(addr, _6planeAddr.rawIpData(), sizeof(struct sockaddr_storage));
-}
-#ifdef SDK_JNI
-JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_16plane_1addr(
-	JNIEnv *env, jobject thisObj, jlong nwid, jlong nodeId, jobject addr)
-{
-	struct sockaddr_storage ss;
-	zts_get_6plane_addr(&ss, nwid, nodeId);
-	ss2zta(env, &ss, addr);
-}
-#endif
-
-void zts_get_rfc4193_addr(struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId)
-{
-	ZeroTier::InetAddress _rfc4193Addr = ZeroTier::InetAddress::makeIpv6rfc4193(nwid,nodeId);
-	memcpy(addr, _rfc4193Addr.rawIpData(), sizeof(struct sockaddr_storage));
-}
-#ifdef SDK_JNI
-JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_1rfc4193_1addr(
-	JNIEnv *env, jobject thisObj, jlong nwid, jlong nodeId, jobject addr)
-{
-	struct sockaddr_storage ss;
-	zts_get_rfc4193_addr(&ss, nwid, nodeId);
-	ss2zta(env, &ss, addr);
-}
-#endif
-*/
-
 
 zts_err_t zts_join(const uint64_t nwid, int blocking)
 {
@@ -351,7 +446,7 @@ zts_err_t zts_join(const uint64_t nwid, int blocking)
 						retval = ZTS_ERR_SERVICE;
 						break;
 					}
-					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+					_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 				}
 			}
 		} else {
@@ -366,7 +461,6 @@ zts_err_t zts_join(const uint64_t nwid, int blocking)
 			if (zt1Service) {
 				zt1Service->getNode()->join(nwid, NULL, NULL);
 			}
-			VirtualTapManager::update_service_references((void*)zt1Service);
 		}
 		_service_lock.unlock();
 		_hibernate_if_needed();
@@ -394,7 +488,7 @@ zts_err_t zts_leave(const uint64_t nwid, int blocking)
 					retval = ZTS_ERR_SERVICE;
 					break;
 				}
-				api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+				_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 			}
 		}
 	} else {
@@ -506,7 +600,7 @@ JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_ready(
 }
 #endif
 
-zts_err_t zts_start(const char *path, int blocking = false)
+zts_err_t zts_start_with_callback(const char *path, void (*callback)(uint64_t, int), int blocking)
 {
 	_startup_lock.lock();
 	zts_err_t retval = ZTS_ERR_OK;
@@ -521,15 +615,31 @@ zts_err_t zts_start(const char *path, int blocking = false)
 	if (!path) {
 		retval = ZTS_ERR_INVALID_ARG;
 	}
+
+    _userCallbackFunc = callback;
+
 	if (retval == ZTS_ERR_OK) {
 		homeDir = path;
 #if defined(_WIN32)
 		// initialize WinSock. Used in Phy for loopback pipe
 		WSAStartup(MAKEWORD(2, 2), &wsaData);
-		HANDLE thr = CreateThread(NULL, 0, _zts_start_service, NULL, 0, NULL);
+		HANDLE serviceThread = CreateThread(NULL, 0, _zts_start_service, NULL, 0, NULL);
+		if (_is_callback_registered()) {
+			HANDLE callbackThread = CreateThread(NULL, 0, _zts_monitor_callback_conditions, NULL, 0, NULL);
+		}
+		// TODO: Add thread names on Windows (optional)
 #else
 		_startupError = false;
 		retval = pthread_create(&service_thread, NULL, _zts_start_service, NULL);
+#if defined(__linux__)
+		pthread_setname_np(service_thread, ZTS_SERVICE_THREAD_NAME);
+#endif
+		if (_is_callback_registered()) {
+			retval = pthread_create(&callback_thread, NULL, _zts_monitor_callback_conditions, NULL);
+#if defined(__linux__)
+			pthread_setname_np(callback_thread, ZTS_EVENT_CALLBACK_THREAD_NAME);
+#endif	
+		}	
 		// Wait for confirmation that the ZT service has been initialized, 
 		// this wait condition is so brief and so rarely used that it should be
 		// acceptable even in a non-blocking context.
@@ -539,7 +649,7 @@ zts_err_t zts_start(const char *path, int blocking = false)
 				retval = ZTS_ERR_SERVICE;
 				break;
 			}
-			api_sleep(10);
+			_api_sleep(10);
 		}
 #endif
 		if (blocking && retval == ZTS_ERR_OK) {
@@ -551,7 +661,7 @@ zts_err_t zts_start(const char *path, int blocking = false)
 					retval = ZTS_ERR_SERVICE;
 					break;
 				}
-				api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+				_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 			}
 			if (retval == ZTS_ERR_OK) {
 				// waiting for node address assignment
@@ -560,7 +670,7 @@ zts_err_t zts_start(const char *path, int blocking = false)
 						retval = ZTS_ERR_SERVICE;
 						break;
 					}
-					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+					_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 				}
 			}
 			if (retval == ZTS_ERR_OK) {
@@ -575,7 +685,7 @@ zts_err_t zts_start(const char *path, int blocking = false)
 						// Node is fully online
 						break;
 					}
-					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+					_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 					_service_lock.unlock();
 				}
 				_service_lock.unlock();
@@ -587,6 +697,35 @@ zts_err_t zts_start(const char *path, int blocking = false)
 	_hibernate_if_needed();
 	return retval;
 }
+
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_start_1with_1callback(
+	JNIEnv *env, jobject thisObj, jstring path, jobject userCallbackClass)
+{
+	jclass eventListenerClass = env->GetObjectClass(userCallbackClass);
+	if(eventListenerClass == NULL) {
+		DEBUG_ERROR("Couldn't find class for ZeroTierEventListener instance");
+		return;
+	}
+	jmethodID eventListenerCallbackMethod = env->GetMethodID(eventListenerClass, "onZeroTierEvent", "(JI)V");
+	if(eventListenerCallbackMethod == NULL) {
+		DEBUG_ERROR("Couldn't find onZeroTierEvent method");
+		return;
+	}
+	objRef = env->NewGlobalRef(userCallbackClass); // Reference used for later calls
+	_userCallbackMethodRef = eventListenerCallbackMethod;
+	if (path) {
+		const char* utf_string = env->GetStringUTFChars(path, NULL);
+		zts_start_with_callback(utf_string, NULL, false);
+		env->ReleaseStringUTFChars(path, utf_string);
+	}
+}
+#endif
+
+zts_err_t zts_start(const char *path, int blocking = false)
+{
+	return zts_start_with_callback(path, NULL, blocking);
+}
 #ifdef SDK_JNI
 JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_start(
 	JNIEnv *env, jobject thisObj, jstring path, jboolean blocking)
@@ -611,7 +750,7 @@ zts_err_t zts_startjoin(const char *path, const uint64_t nwid)
 			break;
 		}
 		catch( ... ) {
-			api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+			_api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
 			retval = ZTS_ERR_SERVICE;
 		}
 	}
@@ -653,6 +792,7 @@ zts_err_t zts_stop(int blocking)
 		pthread_join(service_thread, NULL);
 	}
 	_hibernate_if_needed();
+	_clear_registered_callback();
 	return retval;
 }
 #ifdef SDK_JNI
@@ -780,7 +920,7 @@ zts_err_t zts_get_network_details(uint64_t nwid, struct zts_network_details *nd)
 		retval = ZTS_ERR_SERVICE;
 	}
 	if (retval == ZTS_ERR_OK) {
-		VirtualTapManager::get_network_details(nwid, nd);
+		VirtualTapManager::get_network_details(zt1Service, nwid, nd);
 	}
 	return retval;
 }
@@ -797,7 +937,7 @@ zts_err_t zts_get_all_network_details(struct zts_network_details *nds, int *num)
 		retval = ZTS_ERR_SERVICE;
 	}
 	if (retval == ZTS_ERR_OK) {
-		VirtualTapManager::get_all_network_details(nds, num);
+		VirtualTapManager::get_all_network_details(zt1Service, nds, num);
 	}
 	return retval;	
 }

src/VirtualTap.cpp

@@ -35,6 +35,9 @@
 #include "Node.hpp"
 #include "OSUtils.hpp"
 
+#include "Constants.hpp" // libzt
+extern void _push_callback_event(uint64_t nwid, int eventCode);
+
 #include "Mutex.hpp"
 #include "VirtualTapManager.hpp"
 #include "lwIP.h"
@@ -45,6 +48,9 @@
 
 namespace ZeroTier {
 
+extern OneService *zt1Service;
+extern void (*_userCallbackFunc)(uint64_t, int);
+
 VirtualTap::VirtualTap(
 	const char *homePath,
 	const MAC &mac,
@@ -79,6 +85,7 @@ VirtualTap::VirtualTap(
 
 VirtualTap::~VirtualTap()
 {
+	_push_callback_event(_nwid, ZTS_EVENT_NETWORK_DOWN);
 	lwip_driver_set_tap_interfaces_down(this);
 	_run = false;
 	::write(_shutdownSignalPipe[1],"\0",1);
@@ -88,6 +95,11 @@ VirtualTap::~VirtualTap()
 	::close(_shutdownSignalPipe[1]);
 }
 
+void VirtualTap::lastConfigUpdate(uint64_t lastConfigUpdateTime)
+{
+	_lastConfigUpdateTime = lastConfigUpdateTime;
+}
+
 void VirtualTap::setEnabled(bool en)
 {
 	_enabled = en;
@@ -106,7 +118,6 @@ void VirtualTap::registerIpWithStack(const InetAddress &ip)
 bool VirtualTap::addIp(const InetAddress &ip)
 {
 	char ipbuf[INET6_ADDRSTRLEN];
-	// DEBUG_INFO("addr=%s, nwid=%llx", ip.toString(ipbuf), (unsigned long long)_nwid);
 	Mutex::Lock _l(_ips_m);
 	registerIpWithStack(ip);
 	if (std::find(_ips.begin(),_ips.end(),ip) == _ips.end()) {
@@ -152,10 +163,10 @@ std::string VirtualTap::deviceName() const
 
 std::string VirtualTap::nodeId() const
 {
-	if (zt1ServiceRef) {
+	if (zt1Service) {
 		char id[ZTS_ID_LEN];
 		memset(id, 0, sizeof(id));
-		sprintf(id, "%llx", (unsigned long long)((OneService *)zt1ServiceRef)->getNode()->address());
+		sprintf(id, "%llx", (unsigned long long)((OneService *)zt1Service)->getNode()->address());
 		return std::string(id);
 	}
 	else {
@@ -207,6 +218,12 @@ void VirtualTap::threadMain()
 	FD_ZERO(&readfds);
 	FD_ZERO(&nullfds);
 	int nfds = (int)std::max(_shutdownSignalPipe[0],0) + 1;
+#if defined(__linux__)
+	pthread_setname_np(pthread_self(), vtap_full_name);
+#endif
+#if defined(__APPLE__)
+	pthread_setname_np(vtap_full_name);
+#endif
 	while (true) {
 		FD_SET(_shutdownSignalPipe[0],&readfds);
 		select(nfds,&readfds,&nullfds,&nullfds,&tv);
@@ -231,7 +248,7 @@ void VirtualTap::threadMain()
 void VirtualTap::Housekeeping()
 {
 	Mutex::Lock _l(_tcpconns_m);
-	OneService *service = ((OneService *)zt1ServiceRef);
+	OneService *service = ((OneService *)zt1Service);
 	if (!service) {
 		return;
 	}

src/VirtualTapManager.cpp

@@ -0,0 +1,116 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Management of virtual tap interfaces
+ */
+
+#include "VirtualTap.hpp"
+#include "VirtualTapManager.hpp"
+#include "ServiceControls.hpp"
+#include "OneService.hpp"
+
+namespace ZeroTier {
+
+extern std::map<uint64_t, VirtualTap*> vtapMap;
+extern Mutex _vtaps_lock;
+extern void (*_userCallbackFunc)(uint64_t, int);
+
+class VirtualTap;
+
+void VirtualTapManager::add_tap(VirtualTap *tap) {
+    _vtaps_lock.lock();
+    vtapMap[tap->_nwid] = tap;
+    _vtaps_lock.unlock();
+}
+
+VirtualTap *VirtualTapManager::getTapByNWID(uint64_t nwid) {
+    _vtaps_lock.lock();
+    VirtualTap *s, *tap = vtapMap[nwid];
+    _vtaps_lock.unlock();
+    return tap;
+}
+
+size_t VirtualTapManager::get_vtaps_size() {
+    size_t sz;
+    _vtaps_lock.lock();
+    sz = vtapMap.size();
+    _vtaps_lock.unlock();
+    return sz;
+}
+
+void VirtualTapManager::remove_by_nwid(uint64_t nwid) {
+    _vtaps_lock.lock();
+    vtapMap.erase(nwid);
+    _vtaps_lock.unlock();
+}
+
+void VirtualTapManager::clear() {
+    _vtaps_lock.lock();
+    vtapMap.clear();
+    _vtaps_lock.unlock();
+}
+
+void VirtualTapManager::get_network_details_helper(void *zt1ServiceRef, uint64_t nwid, struct zts_network_details *nd)
+{
+    socklen_t addrlen;
+    VirtualTap *tap = vtapMap[nwid];
+    nd->nwid = tap->_nwid;
+    nd->mtu = tap->_mtu;
+    // assigned addresses
+    nd->num_addresses = tap->_ips.size() < ZTS_MAX_ASSIGNED_ADDRESSES ? tap->_ips.size() : ZTS_MAX_ASSIGNED_ADDRESSES;
+    for (int j=0; j<nd->num_addresses; j++) {
+        addrlen = tap->_ips[j].isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+        memcpy(&(nd->addr[j]), &(tap->_ips[j]), addrlen);
+    }
+    // routes
+    nd->num_routes = ZTS_MAX_NETWORK_ROUTES;
+    OneService *zt1Service = (OneService*)zt1ServiceRef;
+    zt1Service->getRoutes(nwid, (ZT_VirtualNetworkRoute*)&(nd->routes)[0], &(nd->num_routes));
+}
+
+void VirtualTapManager::get_network_details(void *zt1ServiceRef, uint64_t nwid, struct zts_network_details *nd) {
+    _vtaps_lock.lock();
+    get_network_details_helper(zt1ServiceRef, nwid, nd);
+    _vtaps_lock.unlock();
+}
+
+void VirtualTapManager::get_all_network_details(void *zt1ServiceRef, struct zts_network_details *nds, int *num) {
+
+    _vtaps_lock.lock();
+    *num = vtapMap.size();
+    int idx = 0;
+    std::map<uint64_t, VirtualTap*>::iterator it;
+    for (it = vtapMap.begin(); it != vtapMap.end(); it++) {
+        get_network_details(zt1ServiceRef, it->first, &nds[idx]);
+        idx++;
+    }
+    _vtaps_lock.unlock();
+}
+
+} // namespace ZeroTier

src/libzt.cpp

@@ -48,6 +48,7 @@
 
 #ifdef __cplusplus
 extern "C" {
+
 #endif
 	void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
 	void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
@@ -66,6 +67,9 @@ extern "C" {
 extern "C" {
 #endif
 
+// Custom errno to prevent conflicts with platform's own errno
+int zts_errno;
+
 // lwIP prototypes copied from lwip/src/include/sockets.h
 // Don't call these directly, call zts_* functions instead
 int lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
@@ -116,7 +120,8 @@ int zts_socket(int socket_family, int socket_type, int protocol)
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
 	JNIEnv *env, jobject thisObj, jint family, jint type, jint protocol)
 {
-	return zts_socket(family, type, protocol);
+	zts_err_t retval = zts_socket(family, type, protocol);
+	return retval > -1 ? retval : -(zts_errno); // Encode lwip errno in return value for JNI functions only
 }
 #endif
 
@@ -137,7 +142,8 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_connect(
 	struct sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
 	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-	return zts_connect(fd, (struct sockaddr *)&ss, addrlen);
+	zts_err_t retval = zts_connect(fd, (struct sockaddr *)&ss, addrlen);
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -158,7 +164,8 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_bind(
 	struct sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
 	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-	return zts_bind(fd, (struct sockaddr*)&ss, addrlen);
+	zts_err_t retval = zts_bind(fd, (struct sockaddr*)&ss, addrlen);
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -170,7 +177,8 @@ int zts_listen(int fd, int backlog)
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(
 	JNIEnv *env, jobject thisObj, jint fd, int backlog)
 {
-	return zts_listen(fd, backlog);
+	zts_err_t retval = zts_listen(fd, backlog);
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -184,9 +192,9 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept(
 {
 	struct sockaddr_storage ss;
 	socklen_t addrlen = sizeof(struct sockaddr_storage);
-	int err = zts_accept(fd, (struct sockaddr *)&ss, &addrlen);
+	zts_err_t retval =zts_accept(fd, (struct sockaddr *)&ss, &addrlen);
 	ss2zta(env, &ss, addr);
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -203,9 +211,9 @@ int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)
  {
 	struct sockaddr_storage ss;
 	socklen_t addrlen = sizeof(struct sockaddr_storage);
-	int err = zts_accept4(fd, (struct sockaddr *)&ss, &addrlen, flags);
+	zts_err_t retval = zts_accept4(fd, (struct sockaddr *)&ss, &addrlen, flags);
 	ss2zta(env, &ss, addr);
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 #endif
@@ -230,10 +238,11 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
 		|| optname == ZTS_SO_REUSEPORT
 		|| optname == ZTS_TCP_NODELAY)
 	{
-		jfieldID fid = (*env).GetFieldID(c, "booleanValue", "B");
+		jfieldID fid = (*env).GetFieldID(c, "booleanValue", "Z");
 		optval_int = (int)(*env).GetBooleanField(optval, fid);
 	}
-	if (optname == ZTS_IP_TTL	
+	if (optname == ZTS_IP_TTL
+		|| optname == ZTS_SO_RCVTIMEO
 		|| optname == ZTS_IP_TOS
 		|| optname == ZTS_SO_LINGER
 		|| optname == ZTS_SO_RCVBUF
@@ -242,8 +251,20 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
 		jfieldID fid = (*env).GetFieldID(c, "integerValue", "I");
 		optval_int = (*env).GetIntField(optval, fid);
 	}
-	int optlen = sizeof(optval_int);
+
-	return zts_setsockopt(fd, level, optname, &optval_int, optlen);
+	zts_err_t retval = ZTS_ERR_OK;
+
+	if (optname == ZTS_SO_RCVTIMEO) {
+		struct timeval tv;
+		// Convert milliseconds from setSoTimeout() call to seconds and microseconds
+		tv.tv_usec = optval_int * 1000;
+		tv.tv_sec = optval_int / 1000000;
+		retval = zts_setsockopt(fd, level, optname, &tv, sizeof(tv));
+	}
+	else {
+		retval = zts_setsockopt(fd, level, optname, &optval_int, sizeof(optval_int));
+	}
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -259,33 +280,46 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
 	if (!c) {
 		return ZTS_ERR_INVALID_OP;
 	}
-	int optval_int;
+	int optval_int = 0;
 	int optlen; // Intentionally not used
-	int err = ZTS_ERR_OK;
+
-	err = zts_getsockopt(fd, level, optname, &optval_int, &optlen);
+	zts_err_t retval;
+
+	if (optname == ZTS_SO_RCVTIMEO) {
+		struct timeval tv;
+		optlen = sizeof(tv);
+		retval = zts_getsockopt(fd, level, optname, &tv, &optlen);
+		// Convert seconds and microseconds back to milliseconds
+		optval_int = (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+	}
+	else {
+		retval = zts_getsockopt(fd, level, optname, &optval_int, &optlen);
+	}
+	
 	if (optname == ZTS_SO_BROADCAST
 		|| optname == ZTS_SO_KEEPALIVE
 		|| optname == ZTS_SO_REUSEADDR	
 		|| optname == ZTS_SO_REUSEPORT
 		|| optname == ZTS_TCP_NODELAY)
 	{
-		jfieldID fid = (*env).GetFieldID(c, "isBoolean", "B");
+		jfieldID fid = (*env).GetFieldID(c, "isBoolean", "Z");
 		(*env).SetBooleanField(optval, fid, true);
-		fid = (*env).GetFieldID(c, "booleanValue", "B");
+		fid = (*env).GetFieldID(c, "booleanValue", "Z");
 		(*env).SetBooleanField(optval, fid, (bool)optval_int);
 	}
-	if (optname == ZTS_IP_TTL	
+	if (optname == ZTS_IP_TTL
+		|| optname == ZTS_SO_RCVTIMEO	
 		|| optname == ZTS_IP_TOS
 		|| optname == ZTS_SO_LINGER
 		|| optname == ZTS_SO_RCVBUF
 		|| optname == ZTS_SO_SNDBUF)
 	{
-		jfieldID fid = (*env).GetFieldID(c, "isInteger", "B");
+		jfieldID fid = (*env).GetFieldID(c, "isInteger", "Z");
 		(*env).SetBooleanField(optval, fid, true);
 		fid = (*env).GetFieldID(c, "integerValue", "I");
 		(*env).SetIntField(optval, fid, optval_int);
 	}
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -305,9 +339,9 @@ JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_getsockname(JNIEnv *
 {
 	struct sockaddr_storage ss;
 	socklen_t addrlen = sizeof(struct sockaddr_storage);
-	int err = zts_getsockname(fd, (struct sockaddr *)&ss, &addrlen);
+	zts_err_t retval =zts_getsockname(fd, (struct sockaddr *)&ss, &addrlen);
-	ss2zta(env, &ss, addr);
+	ss2zta(env, &ss, addr);	 
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -326,9 +360,9 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getpeername(JNIEnv *env,
 	jint fd, jobject addr)
 {
 	struct sockaddr_storage ss;
-	int err = zts_getpeername(fd, (struct sockaddr *)&ss, (socklen_t *)sizeof(struct sockaddr_storage));
+	zts_err_t retval = zts_getpeername(fd, (struct sockaddr *)&ss, (socklen_t *)sizeof(struct sockaddr_storage));
-	ss2zta(env, &ss, addr);
+	ss2zta(env, &ss, addr);	 
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -394,34 +428,34 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_select(JNIEnv *env, jobj
 {
 	struct timeval _timeout;
 	_timeout.tv_sec  = timeout_sec;
-		_timeout.tv_usec = timeout_usec;
+	_timeout.tv_usec = timeout_usec;
-		fd_set _readfds, _writefds, _exceptfds;   		
+	fd_set _readfds, _writefds, _exceptfds;   		
-		fd_set *r = NULL;
+	fd_set *r = NULL;
-		fd_set *w = NULL;
+	fd_set *w = NULL;
-		fd_set *e = NULL;
+	fd_set *e = NULL;
-		if (readfds) {
+	if (readfds) {
-			r = &_readfds;
+		r = &_readfds;
-			ztfdset2fdset(env, nfds, readfds, &_readfds);
+		ztfdset2fdset(env, nfds, readfds, &_readfds);
-		}
+	}
-		if (writefds) {
+	if (writefds) {
-			w = &_writefds;
+		w = &_writefds;
-			ztfdset2fdset(env, nfds, writefds, &_writefds);
+		ztfdset2fdset(env, nfds, writefds, &_writefds);
-		}
+	}
-		if (exceptfds) {
+	if (exceptfds) {
-			e = &_exceptfds;
+		e = &_exceptfds;
-			ztfdset2fdset(env, nfds, exceptfds, &_exceptfds);
+		ztfdset2fdset(env, nfds, exceptfds, &_exceptfds);
-		}
+	}
-	int err = zts_select(nfds, r, w, e, &_timeout);
+	zts_err_t retval = zts_select(nfds, r, w, e, &_timeout);
 	if (readfds) {
 		fdset2ztfdset(env, nfds, &_readfds, readfds);
 	}
 	if (writefds) {
-			fdset2ztfdset(env, nfds, &_writefds, writefds);
+		fdset2ztfdset(env, nfds, &_writefds, writefds);
-		}
+	}
-		if (exceptfds) {
+	if (exceptfds) {
-			fdset2ztfdset(env, nfds, &_exceptfds, exceptfds);
+		fdset2ztfdset(env, nfds, &_exceptfds, exceptfds);
-		}
+	}	 
-	return err;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -445,7 +479,8 @@ int zts_fcntl(int fd, int cmd, int flags)
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_fcntl(
 	JNIEnv *env, jobject thisObj, jint fd, jint cmd, jint flags)
 {
-	return zts_fcntl(fd, cmd, flags);
+	zts_err_t retval = zts_fcntl(fd, cmd, flags);
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -479,7 +514,7 @@ JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
 		DEBUG_ERROR("FIONBIO");
 		retval = zts_ioctl(fd, request, &meaninglessVariable);		
 	}
-	return retval;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -495,9 +530,9 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_send(
 	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, int flags)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int w = zts_send(fd, data, env->GetArrayLength(buf), flags);
+	zts_err_t retval = zts_send(fd, data, env->GetArrayLength(buf), flags);
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
-	return w;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -520,9 +555,9 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(
 	struct sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
 	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-	int w = zts_sendto(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, addrlen);
+	zts_err_t retval = zts_sendto(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, addrlen);
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
-	return w;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -545,9 +580,9 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recv(JNIEnv *env, jobjec
 	jint fd, jbyteArray buf, jint flags)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int r = zts_recv(fd, data, env->GetArrayLength(buf), flags);
+	zts_err_t retval = zts_recv(fd, data, env->GetArrayLength(buf), flags);
 	env->ReleasePrimitiveArrayCritical(buf, data, 0);
-	return r;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -566,10 +601,10 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recvfrom(
 	socklen_t addrlen = sizeof(struct sockaddr_storage);
 	struct sockaddr_storage ss;
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int r = zts_recvfrom(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, &addrlen);
+	zts_err_t retval = zts_recvfrom(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, &addrlen);
 	env->ReleasePrimitiveArrayCritical(buf, data, 0);
-	ss2zta(env, &ss, addr);
+	ss2zta(env, &ss, addr);	 
-	return r;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -600,25 +635,25 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read(JNIEnv *env, jobjec
 	jint fd, jbyteArray buf)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int r = zts_read(fd, data, env->GetArrayLength(buf));
+	zts_err_t retval = zts_read(fd, data, env->GetArrayLength(buf));
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
-	return r;
+	return retval > -1 ? retval : -(zts_errno);
 }
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read_1offset(JNIEnv *env, jobject thisObj,
 	jint fd, jbyteArray buf, jint offset, jint len)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int r = zts_read_offset(fd, data, offset, len);
+	zts_err_t retval = zts_read_offset(fd, data, offset, len);
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
-	return r;
+	return retval > -1 ? retval : -(zts_errno);
 }
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read_1length(JNIEnv *env, jobject thisObj,
 	jint fd, jbyteArray buf, jint len)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int r = zts_read(fd, data, len);
+	zts_err_t retval = zts_read(fd, data, len);
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	
-	return r;
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
@@ -634,22 +669,23 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write__IB(JNIEnv *env, j
 	jint fd, jbyteArray buf)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int w = zts_write(fd, data, env->GetArrayLength(buf));
+	zts_err_t retval = zts_write(fd, data, env->GetArrayLength(buf));
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
-	return w;
+	return retval > -1 ? retval : -(zts_errno);
 }
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1offset(JNIEnv *env, jobject thisObj,
 	jint fd, jbyteArray buf, jint offset, jint len)
 {
 	void *data = env->GetPrimitiveArrayCritical(&(buf[offset]), NULL); // PENDING: check?
-	int w = zts_write(fd, data, len);
+	zts_err_t retval = zts_write(fd, data, len);
-	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);	
-	return w;
+	return retval > -1 ? retval : -(zts_errno);
 }
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1byte(JNIEnv *env, jobject thisObj,
 	jint fd, jbyte buf)
 {
-	return zts_write(fd, &buf, 1);
+	zts_err_t retval = zts_write(fd, &buf, 1);
+	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 

src/lwIP.cpp

@@ -31,6 +31,7 @@
  */
 
 #include <vector>
+#include <queue>
 
 #include "MAC.hpp"
 
@@ -46,7 +47,6 @@
 #include "lwip/memp.h"
 #include "lwip/sys.h"
 #include "lwip/tcp.h"
-#include "lwip/priv/tcp_priv.h" /* for tcp_debug_print_pcbs() */
 #include "lwip/timeouts.h"
 #include "lwip/stats.h"
 #include "lwip/ethip6.h"
@@ -65,9 +65,9 @@ void ms_sleep(unsigned long ms)
 }
 #endif
 
+std::queue<struct pbuf *> rx_queue;
+
 ZeroTier::Mutex _rx_input_lock_m;
-struct pbuf* lwip_frame_rxbuf[LWIP_MAX_GUARDED_RX_BUF_SZ];
-int lwip_frame_rxbuf_tot = 0;
 
 bool main_loop_exited = false;
 bool lwip_driver_initialized = false;
@@ -103,14 +103,19 @@ void my_tcpip_callback(void *arg)
 	if (main_loop_exited) {
 		return;
 	}
-	ZeroTier::Mutex::Lock _l(_rx_input_lock_m);
+	err_t err = ERR_OK;
 	int loop_score = LWIP_FRAMES_HANDLED_PER_CORE_CALL; // max num of packets to read per polling call
 	// TODO: Optimize (use Ringbuffer)
-	int pkt_num = 0;
+	while (loop_score > 0) {
-	int count_initial = lwip_frame_rxbuf_tot;
+		// TODO: Swap this block out for a thread-safe container
-	while (lwip_frame_rxbuf_tot > 0 && loop_score > 0) {
+		_rx_input_lock_m.lock();
-		struct pbuf *p = lwip_frame_rxbuf[pkt_num];
+		if (rx_queue.size() == 0) {
-		pkt_num++;
+			_rx_input_lock_m.unlock();
+			return;
+		}
+		struct pbuf *p = rx_queue.front();
+		rx_queue.pop();
+		_rx_input_lock_m.unlock();
 		// Packet routing logic. Inputs packet into correct lwip netif interface depending on protocol type
 		struct ip_hdr *iphdr;
 		switch (((struct eth_hdr *)p->payload)->type)
@@ -120,8 +125,8 @@ void my_tcpip_callback(void *arg)
 				for (size_t i=0; i<lwip_netifs.size(); i++) {
 					if (lwip_netifs[i]->output_ip6 && 
 						lwip_netifs[i]->output_ip6 == ethip6_output) {
-						if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
+						if ((err = lwip_netifs[i]->input(p, lwip_netifs[i])) != ERR_OK) {
-							DEBUG_ERROR("packet input error (ipv6, p=%p, netif=%p)", p, &lwip_netifs[i]);
+							DEBUG_ERROR("packet input error (ipv6, p=%p, netif=%p)=%d", p, &lwip_netifs[i], err);
 							break;
 						}
 					}
@@ -134,8 +139,8 @@ void my_tcpip_callback(void *arg)
 						lwip_netifs[i]->output == etharp_output) {
 						if (lwip_netifs[i]->ip_addr.u_addr.ip4.addr == iphdr->dest.addr || 
 							ip4_addr_isbroadcast_u32(iphdr->dest.addr, lwip_netifs[i])) {
-							if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
+							if ((err = lwip_netifs[i]->input(p, lwip_netifs[i])) != ERR_OK) {
-								DEBUG_ERROR("packet input error (ipv4, p=%p, netif=%p)", p, &lwip_netifs[i]);
+								DEBUG_ERROR("packet input error (ipv4, p=%p, netif=%p)=%d", p, &lwip_netifs[i], err);
 								break;
 							}
 						}
@@ -146,8 +151,8 @@ void my_tcpip_callback(void *arg)
 				for (size_t i=0; i<lwip_netifs.size(); i++) {
 					if (lwip_netifs[i]->state) {
 						pbuf_ref(p);
-						if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
+						if ((err = lwip_netifs[i]->input(p, lwip_netifs[i])) != ERR_OK) {
-							DEBUG_ERROR("packet input error (arp, p=%p, netif=%p)", p, &lwip_netifs[i]);
+							DEBUG_ERROR("packet input error (arp, p=%p, netif=%p)=%d", p, &lwip_netifs[i], err);
 						}
 						break;
 					}
@@ -157,19 +162,19 @@ void my_tcpip_callback(void *arg)
 			default:
 				break;
 		}
-		lwip_frame_rxbuf_tot--;
 		loop_score--;
 	}
-	int count_final = lwip_frame_rxbuf_tot;
-	// Move pbuf frame pointer address buffer by the number of frames successfully fed into the stack core
-	if (count_initial - count_final > 0) {
-		memmove(lwip_frame_rxbuf, lwip_frame_rxbuf + count_final, count_initial - count_final);
-	}
 }
 
 // main thread which starts the initialization process
 static void main_lwip_driver_loop(void *arg)
 {
+#if defined(__linux__)
+	pthread_setname_np(pthread_self(), "lwip_driver_loop");
+#endif
+#if defined(__APPLE__)
+	pthread_setname_np("lwip_driver_loop");
+#endif
 	sys_sem_t sem;
 	LWIP_UNUSED_ARG(arg);
 	if (sys_sem_new(&sem, 0) != ERR_OK) {
@@ -178,8 +183,6 @@ static void main_lwip_driver_loop(void *arg)
 	tcpip_init(tcpip_init_done, &sem);
 	has_already_been_initialized = true;
 	sys_sem_wait(&sem);
-	//DEBUG_INFO("stack thread init complete");
-
 	while(lwip_driver_initialized) {
 #if defined(_WIN32)
 		ms_sleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*hibernationDelayMultiplier);
@@ -301,12 +304,19 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
 			ZeroTier::Utils::ntoh(ethhdr->type), flagbuf);
 	}
+	
+	
 	return ERR_OK;
 }
 
 void lwip_eth_rx(ZeroTier::VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
 	const void *data, unsigned int len)
 {
+	if (!lwip_netifs.size()) {
+		DEBUG_ERROR("there are no netifs set up to handle this packet. ignoring.");
+		return;
+	}
+
 	struct pbuf *p,*q;
 	struct eth_hdr ethhdr;
 	from.copyTo(ethhdr.src.addr, 6);
@@ -326,41 +336,35 @@ void lwip_eth_rx(ZeroTier::VirtualTap *tap, const ZeroTier::MAC &from, const Zer
 		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
 			ZeroTier::Utils::ntoh(ethhdr.type), flagbuf);
 	}
-
+	
-	p = pbuf_alloc(PBUF_RAW, len+sizeof(struct eth_hdr), PBUF_POOL);
+	p = pbuf_alloc(PBUF_RAW, len+sizeof(struct eth_hdr), PBUF_RAM);
-	if (p != NULL) {
+	if (!p) {
-		const char *dataptr = reinterpret_cast<const char *>(data);
+		DEBUG_ERROR("dropped packet: unable to allocate memory for pbuf");
-		// First pbuf gets ethernet header at start
-		q = p;
-		if (q->len < sizeof(ethhdr)) {
-			DEBUG_ERROR("dropped packet: first pbuf smaller than ethernet header");
-			return;
-		}
-		memcpy(q->payload,&ethhdr,sizeof(ethhdr));
-		memcpy((char*)q->payload + sizeof(ethhdr),dataptr,q->len - sizeof(ethhdr));
-		dataptr += q->len - sizeof(ethhdr);
-		// Remaining pbufs (if any) get rest of data
-		while ((q = q->next)) {
-			memcpy(q->payload,dataptr,q->len);
-			dataptr += q->len;
-		}
-	}
-	else {
-		DEBUG_ERROR("dropped packet: no pbufs available");
 		return;
 	}
-	if (!lwip_netifs.size()) {
+	// First pbuf gets ethernet header at start
-		DEBUG_ERROR("there are no netifs set up to handle this packet. ignoring.");
+	q = p;
+	if (q->len < sizeof(ethhdr)) {
+		DEBUG_ERROR("dropped packet: first pbuf smaller than ethernet header");
 		return;
 	}
-	ZeroTier::Mutex::Lock _l(_rx_input_lock_m);
+	const char *dataptr = reinterpret_cast<const char *>(data);
-	if (lwip_frame_rxbuf_tot == LWIP_MAX_GUARDED_RX_BUF_SZ) {
+	memcpy(q->payload,&ethhdr,sizeof(ethhdr));
-		DEBUG_ERROR("dropped packet -- guarded receive buffer full, adjust MAX_GUARDED_RX_BUF_SZ or LWIP_GUARDED_BUF_CHECK_INTERVAL");
+	int remainingPayloadSpace = q->len - sizeof(ethhdr);
+	memcpy((char*)q->payload + sizeof(ethhdr),dataptr,remainingPayloadSpace);
+	dataptr += remainingPayloadSpace;
+	// Remaining pbufs (if any) get rest of data
+	while ((q = q->next)) {
+		memcpy(q->payload,dataptr,q->len);
+		dataptr += q->len;
+	}
+	_rx_input_lock_m.lock();
+	if (rx_queue.size() >= LWIP_MAX_GUARDED_RX_BUF_SZ) {
+		DEBUG_INFO("dropped packet: rx_queue is full (>= %d)", LWIP_MAX_GUARDED_RX_BUF_SZ);
 		return;
 	}
-	//pbuf_ref(p); // Increment reference to allow user application to copy data from buffer -- Will be automatically deallocated by socket API
+	rx_queue.push(p);
-	lwip_frame_rxbuf[lwip_frame_rxbuf_tot] = p;
+	_rx_input_lock_m.unlock();
-	lwip_frame_rxbuf_tot += 1;
 }
 
 /*
@@ -377,9 +381,9 @@ void lwip_start_dhcp(void *netif)
 #endif
 }
 
+/*
 static void netif_status_callback(struct netif *netif)
 {
-	/*
 	DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n",
 		netif,
 		netif->name[0],
@@ -397,8 +401,8 @@ static void netif_status_callback(struct netif *netif)
 		netif->state,
 		netif->flags
 	);
-	*/
 }
+*/
 
 ZeroTier::MAC _mac;
 
@@ -455,7 +459,7 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 		IP4_ADDR(&gw,127,0,0,1);
 		ipaddr.addr = *((u32_t *)ip.rawIpData());
 		netmask.addr = *((u32_t *)ip.netmask().rawIpData());
-		netif_set_status_callback(lwipdev, netif_status_callback);
+		//netif_set_status_callback(lwipdev, netif_status_callback);
 		netif_add(lwipdev, &ipaddr, &netmask, &gw, NULL, netif_init_4, tcpip_input);
 		lwipdev->state = tapref;
 		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
@@ -475,7 +479,7 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 		netif_create_ip6_linklocal_address(lwipdev, 1);
 		netif_ip6_addr_set_state(lwipdev, 0, IP6_ADDR_TENTATIVE);
 		netif_ip6_addr_set_state(lwipdev, 1, IP6_ADDR_TENTATIVE);
-		netif_set_status_callback(lwipdev, netif_status_callback);
+		//netif_set_status_callback(lwipdev, netif_status_callback);
 		netif_set_default(lwipdev);
 		netif_set_up(lwipdev);
 		netif_set_link_up(lwipdev);