Re-work of thread model

2019-02-06 22:00:39 -08:00
parent 292fcdda2c
commit 2fdcf025e1
138 changed files with 7567 additions and 15053 deletions
--- a/src/Controls.cpp
+++ b/src/Controls.cpp
@@ -0,0 +1,760 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * ZeroTier service controls
+ */
+
+#include <queue>
+
+#include "Service.hpp"
+#include "Node.hpp"
+#include "ZeroTierOne.h"
+
+#include "OSUtils.hpp"
+#include "VirtualTap.hpp"
+#include "Debug.hpp"
+#include "concurrentqueue.h"
+
+#include "libzt.h"
+
+#if defined(_WIN32)
+WSADATA wsaData;
+#include <Windows.h>
+#endif
+
+#ifdef SDK_JNI
+#include <jni.h>
+#endif
+
+namespace ZeroTier {
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// Custom errno to prevent conflicts with platform's own errno
+int zts_errno;
+#ifdef __cplusplus
+}
+#endif
+
+struct serviceParameters
+{
+	int port;
+	std::string path;
+};
+
+int _port;
+std::string _path;
+
+/*
+ * A lock used to protect any call which relies on the presence of a valid pointer
+ * to the ZeroTier service.
+ */
+Mutex _service_lock;
+
+/*
+ * A lock which protects flags and state variables used during the startup and
+ * shutdown phase.
+ */
+Mutex _startup_lock;
+
+/*
+ * A lock used to protect callback method pointers. With a coarser-grained lock it
+ * would be possible for one thread to alter the callback method pointer causing 
+ * undefined behaviour.
+ */
+Mutex _callback_lock;
+
+bool _freeHasBeenCalled = false;
+bool _run_service = false;
+bool _run_callbacks = false;
+bool _run_lwip_tcpip = false;
+
+//bool _startupError = false;
+
+pthread_t service_thread;
+pthread_t callback_thread;
+
+// Global reference to ZeroTier service
+OneService *service;
+
+// User-provided callback for ZeroTier events
+#ifdef SDK_JNI
+	// Global references to JNI objects and VM kept for future callbacks
+	static JavaVM *jvm = NULL;
+	static jobject objRef = NULL;
+	static jmethodID _userCallbackMethodRef = NULL;
+#endif
+
+void (*_userEventCallbackFunc)(uint64_t, int);
+
+extern moodycamel::ConcurrentQueue<std::pair<uint64_t, int>*> _callbackMsgQueue;
+
+//////////////////////////////////////////////////////////////////////////////
+// Internal ZeroTier Service Controls (user application shall not use these)//
+//////////////////////////////////////////////////////////////////////////////
+
+void postEvent(uint64_t id, int eventCode)
+{
+    // Queue callback event messages from other threads (such as lwIP driver)
+    _callbackMsgQueue.enqueue(new std::pair<uint64_t, int>(id, eventCode));
+}
+
+void _process_callback_event_helper(uint64_t nwid, int eventCode)
+{
+#ifdef SDK_JNI
+	if(_userCallbackMethodRef) {
+		JNIEnv *env;
+		jint rs = jvm->AttachCurrentThread(&env, NULL);
+		assert (rs == JNI_OK);
+		env->CallVoidMethod(objRef, _userCallbackMethodRef, nwid, eventCode);
+	}
+#else
+	if (_userEventCallbackFunc) {
+		_userEventCallbackFunc(nwid, eventCode);
+	}
+#endif
+}
+
+void _process_callback_event(uint64_t nwid, int eventCode)
+{
+	_callback_lock.lock();
+	_process_callback_event_helper(nwid, eventCode);
+	_callback_lock.unlock();
+}
+
+bool _is_callback_registered()
+{
+	_callback_lock.lock();
+	bool retval = false;
+#ifdef SDK_JNI
+	retval = (jvm && objRef && _userCallbackMethodRef);
+#else
+	retval = _userEventCallbackFunc;
+#endif
+	_callback_lock.unlock();
+	return retval;
+}
+
+void _clear_registered_callback()
+{
+	_callback_lock.lock();
+#ifdef SDK_JNI
+	objRef = NULL;
+    _userCallbackMethodRef = NULL;
+#else
+	_userEventCallbackFunc = NULL;
+#endif
+	_callback_lock.unlock();
+}
+
+int __zts_node_online()
+{
+	return service && service->getNode() && service->getNode()->online();
+}
+
+int __zts_can_perform_service_operation()
+{
+	return service
+		&& service->isRunning()
+		&& service->getNode()
+		&& service->getNode()->online()
+		&& !_freeHasBeenCalled;
+}
+
+void _api_sleep(int interval_ms)
+{
+#if defined(_WIN32)
+	Sleep(interval_ms);
+#else
+	struct timespec sleepValue = {0};
+	sleepValue.tv_nsec = interval_ms * 500000;
+	nanosleep(&sleepValue, NULL);
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// Callback thread                                                          //
+//////////////////////////////////////////////////////////////////////////////
+
+#if defined(_WIN32)
+DWORD WINAPI _zts_run_callbacks(LPVOID thread_id)
+#else
+void *_zts_run_callbacks(void *thread_id)
+#endif
+{
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_EVENT_CALLBACK_THREAD_NAME);
+#endif
+	while (_run_callbacks || _callbackMsgQueue.size_approx() > 0)
+    {
+        std::pair<uint64_t, int> *msg;
+		for (int j = 0; j != 32; j++) { // TODO: Check size of queue
+			if (_callbackMsgQueue.try_dequeue(msg)) {
+				// DEBUG_INFO("deqeueuing front: %llx,%d", msg->first, msg->second);
+                _process_callback_event(msg->first, msg->second);
+				delete msg;
+			}
+		}
+        _api_sleep(ZTS_CALLBACK_PROCESSING_INTERVAL);
+    }
+    pthread_exit(0);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// Service thread                                                           //
+//////////////////////////////////////////////////////////////////////////////
+
+// Starts a ZeroTier service in the background
+#if defined(_WIN32)
+DWORD WINAPI _zts_run_service(LPVOID arg)
+#else
+void *_zts_run_service(void *arg)
+#endif
+{	
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_SERVICE_THREAD_NAME);
+#endif
+	//struct serviceParameters *params = arg;
+	//DEBUG_INFO("path=%s", params->path.c_str());
+	int err;
+	
+	try {
+		std::vector<std::string> hpsp(OSUtils::split(_path.c_str(), ZT_PATH_SEPARATOR_S,"",""));
+		std::string ptmp;
+		if (_path[0] == ZT_PATH_SEPARATOR) {
+			ptmp.push_back(ZT_PATH_SEPARATOR);
+		}
+		for (std::vector<std::string>::iterator pi(hpsp.begin());pi!=hpsp.end();++pi) {
+			if (ptmp.length() > 0) {
+				ptmp.push_back(ZT_PATH_SEPARATOR);
+			}
+			ptmp.append(*pi);
+			if ((*pi != ".")&&(*pi != "..")) {
+				if (OSUtils::mkdir(ptmp) == false) {
+					DEBUG_ERROR("home path does not exist, and could not create");
+					err = true;
+					perror("error\n");
+				}
+			}
+		}
+		for(;;) {
+			_service_lock.lock();
+			service = OneService::newInstance(_path.c_str(),_port);
+			_service_lock.unlock();
+			switch(service->run()) {
+				case OneService::ONE_STILL_RUNNING:
+				case OneService::ONE_NORMAL_TERMINATION:
+					postEvent((uint64_t)0, ZTS_EVENT_NODE_NORMAL_TERMINATION);
+					break;
+				case OneService::ONE_UNRECOVERABLE_ERROR:
+					DEBUG_ERROR("fatal error: %s", service->fatalErrorMessage().c_str());
+					err = true;
+					postEvent((uint64_t)0, ZTS_EVENT_NODE_UNRECOVERABLE_ERROR);
+					break;
+				case OneService::ONE_IDENTITY_COLLISION: {
+					err = true;
+					delete service;
+					service = (OneService *)0;
+					std::string oldid;
+					OSUtils::readFile((_path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid);
+					if (oldid.length()) {
+						OSUtils::writeFile((_path + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid);
+						OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
+						OSUtils::rm((_path + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
+					}
+					postEvent((uint64_t)0, ZTS_EVENT_NODE_IDENTITY_COLLISION);
+				}	continue; // restart!
+			}
+			break; // terminate loop -- normally we don't keep restarting
+		}
+		_service_lock.lock();
+		_run_service = false;
+		delete service;
+		service = (OneService *)0;
+		_service_lock.unlock();
+		postEvent((uint64_t)0, ZTS_EVENT_NODE_DOWN);
+	} catch ( ... ) {
+		DEBUG_ERROR("unexpected exception starting ZeroTier instance");
+	}
+	//delete params;
+	// TODO: Find a more elegant solution
+	_api_sleep(ZTS_CALLBACK_PROCESSING_INTERVAL*2);
+	_run_callbacks = false;
+	pthread_exit(0);
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Service Controls                                                //
+//////////////////////////////////////////////////////////////////////////////
+
+#ifdef SDK_JNI
+/*
+ * Called from Java, saves a static reference to the VM so it can be used later to call
+ * a user-specified callback method from C.
+ */
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_init(JNIEnv *env, jobject thisObj)
+{
+    jint rs = env->GetJavaVM(&jvm);
+    assert (rs == JNI_OK);
+}
+#endif
+
+zts_err_t zts_join(const uint64_t nwid)
+{
+	Mutex::Lock _l(_service_lock);
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	else {
+		service->join(nwid);
+	}
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_join(
+	JNIEnv *env, jobject thisObj, jlong nwid)
+{
+	return zts_join((uint64_t)nwid);
+}
+#endif
+
+zts_err_t zts_leave(const uint64_t nwid)
+{
+	Mutex::Lock _l(_service_lock);
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	else {
+		service->leave(nwid);
+	}
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_leave(
+	JNIEnv *env, jobject thisObj, jlong nwid)
+{
+	return zts_leave((uint64_t)nwid);
+}
+#endif
+
+zts_err_t zts_leave_all()
+{
+	Mutex::Lock _l(_service_lock);
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	else {
+		service->leaveAll();
+	}
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_orbit(uint64_t moonWorldId, uint64_t moonSeed)
+{
+	Mutex::Lock _l(_service_lock);
+	void *tptr = NULL;
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	} else {
+		service->getNode()->orbit(tptr, moonWorldId, moonSeed);
+	}
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_deorbit(uint64_t moonWorldId)
+{
+	Mutex::Lock _l(_service_lock);
+	void *tptr = NULL;
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	} else {
+		service->getNode()->deorbit(tptr, moonWorldId);
+	}
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_start(const char *path, void (*callback)(uint64_t, int), int port)
+{
+	Mutex::Lock _l(_service_lock);
+	if (service || _run_service) {
+		// Service is already initialized
+		return ZTS_ERR_INVALID_OP;
+	}
+	if (_freeHasBeenCalled) {
+		// Stack (presumably lwIP) has been dismantled, an application restart is required now
+		return ZTS_ERR_INVALID_OP;
+	}
+#ifdef SDK_JNI
+	_userEventCallbackFunc = callback;
+#endif
+	_userEventCallbackFunc = callback;
+	if (!_is_callback_registered()) {
+		// Must have a callback
+		return ZTS_ERR_INVALID_ARG;
+	}
+	if (!path) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	if (port < 0 || port > 0xFFFF) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+
+	_path = std::string(path);
+	_port = port;
+	
+	serviceParameters *params = new serviceParameters();
+
+	/*
+	params->port = port;
+	DEBUG_INFO("path=%s", path);
+	params->path = std::string(path);
+	DEBUG_INFO("path=%s", params->path.c_str());
+
+	if (params->path.length() == 0) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	*/
+
+	int err;
+	int retval = ZTS_ERR_OK;
+	_run_callbacks = true;
+	_run_service = true;
+
+	// Start the ZT service thread
+#if defined(_WIN32)
+	// Initialize WinSock. Used in Phy for loopback pipe
+	WSAStartup(MAKEWORD(2, 2), &wsaData);
+	HANDLE serviceThread = CreateThread(NULL, 0, _zts_run_service, (void*)params, 0, NULL);
+	HANDLE callbackThread = CreateThread(NULL, 0, _zts_run_callbacks, NULL, 0, NULL);
+#endif
+	if ((err = pthread_create(&service_thread, NULL, _zts_run_service, NULL)) != 0) {
+		retval = err;
+	}
+	if ((err = pthread_create(&callback_thread, NULL, _zts_run_callbacks, NULL)) != 0) {
+		retval = err;
+	}
+#if defined(__linux__)
+	pthread_setname_np(service_thread, ZTS_SERVICE_THREAD_NAME);
+	pthread_setname_np(callback_thread, ZTS_EVENT_CALLBACK_THREAD_NAME);
+#endif
+
+	if (retval != ZTS_ERR_OK) {
+		_run_callbacks = false;
+		_run_service = false;
+		_clear_registered_callback();
+		delete params;
+	}
+	return retval;
+}
+
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_start(
+	JNIEnv *env, jobject thisObj, jstring path, jobject callback, jint port)
+{
+	if (!path) {
+		return;
+	}
+	jclass eventListenerClass = env->GetObjectClass(callback);
+	if(eventListenerClass == NULL) {
+		DEBUG_ERROR("Couldn't find class for ZeroTierEventListener instance");
+		return;
+	}
+	jmethodID eventListenerCallbackMethod = env->GetMethodID(eventListenerClass, "onZeroTierEvent", "(JI)V");
+	if(eventListenerCallbackMethod == NULL) {
+		DEBUG_ERROR("Couldn't find onZeroTierEvent method");
+		return;
+	}
+	objRef = env->NewGlobalRef(callback); // Reference used for later calls
+	_userCallbackMethodRef = eventListenerCallbackMethod;
+	const char* utf_string = env->GetStringUTFChars(path, NULL);
+	zts_start(utf_string, NULL, port); // using _userCallbackMethodRef
+	env->ReleaseStringUTFChars(path, utf_string);
+}
+#endif
+
+zts_err_t zts_stop()
+{
+	Mutex::Lock _l(_service_lock);
+	bool didStop = false;
+	if (__zts_can_perform_service_operation()) {
+		_run_service = false;
+		service->terminate();
+		didStop = true;
+#if defined(_WIN32)
+		WSACleanup();
+#endif
+		return ZTS_ERR_OK;
+	}
+	return ZTS_ERR_SERVICE;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_stop(
+	JNIEnv *env, jobject thisObj)
+{
+	zts_stop();
+}
+#endif
+
+zts_err_t zts_free()
+{
+	Mutex::Lock _l(_service_lock);
+	zts_err_t retval = 0;
+	if (_freeHasBeenCalled) {
+		return ZTS_ERR_INVALID_OP;
+	}
+	_freeHasBeenCalled = true;
+	return zts_stop();
+	// TODO: add stack shutdown logic
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_free(
+	JNIEnv *env, jobject thisObj)
+{
+	zts_free();
+}
+#endif
+
+uint64_t zts_get_node_id()
+{
+	Mutex::Lock _l(_service_lock);
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	return service->getNode()->address();
+}
+#ifdef SDK_JNI
+JNIEXPORT jlong JNICALL Java_com_zerotier_libzt_ZeroTier_get_1node_1id(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_get_node_id();
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Peers                                                                    //
+//////////////////////////////////////////////////////////////////////////////
+
+int zts_get_peer_count()
+{
+	Mutex::Lock _l(_service_lock);
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	return service->getNode()->peers()->peerCount;
+}
+#ifdef SDK_JNI
+JNIEXPORT jlong JNICALL Java_com_zerotier_libzt_ZeroTier_get_1peer_1count(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_get_peer_count();
+}
+#endif
+
+int zts_get_peers(struct zts_peer_details *pds, int *num)
+{
+	// TODO: Modernize
+	Mutex::Lock _l(_service_lock);
+	if (!pds || !num) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	ZT_PeerList *pl = service->getNode()->peers();
+	if (pl) {
+		*num = pl->peerCount;
+		for(unsigned long i=0;i<pl->peerCount;++i) {
+			memcpy(&(pds[i]), &(pl->peers[i]), sizeof(struct zts_peer_details));
+		}
+	}
+	service->getNode()->freeQueryResult((void *)pl);
+	return ZTS_ERR_OK;
+}
+#ifdef SDK_JNI
+#endif
+
+int zts_get_peer_status(uint64_t id)
+{
+	Mutex::Lock _l(_service_lock);
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	return service->getPeerStatus(id);
+}
+#ifdef SDK_JNI
+JNIEXPORT jlong JNICALL Java_com_zerotier_libzt_ZeroTier_get_1peer_1status(
+	JNIEnv *env, jobject thisObj, jlong id)
+{
+	return zts_get_peer_status(id);
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Networks                                                                 //
+//////////////////////////////////////////////////////////////////////////////
+
+zts_err_t zts_get_num_joined_networks()
+{
+	Mutex::Lock _l(_service_lock);
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!__zts_can_perform_service_operation()) {
+		return ZTS_ERR_SERVICE;
+	}
+	return service->networkCount();
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_get_1num_1joined_1networks(
+		JNIEnv *env, jobject thisObj)
+{
+	return zts_get_num_joined_networks();
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Network Details                                                          //
+//////////////////////////////////////////////////////////////////////////////
+
+void __get_network_details_helper(uint64_t nwid, struct zts_network_details *nd)
+{
+	/*
+    socklen_t addrlen;
+    VirtualTap *tap = vtapMap[nwid];
+    nd->nwid = tap->_nwid;
+    nd->mtu = tap->_mtu;
+    // assigned addresses
+    nd->num_addresses = tap->_ips.size() < ZTS_MAX_ASSIGNED_ADDRESSES ? tap->_ips.size() : ZTS_MAX_ASSIGNED_ADDRESSES;
+    for (int j=0; j<nd->num_addresses; j++) {
+        addrlen = tap->_ips[j].isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+        memcpy(&(nd->addr[j]), &(tap->_ips[j]), addrlen);
+    }
+    // routes
+    nd->num_routes = ZTS_MAX_NETWORK_ROUTES;;
+    service->getRoutes(nwid, (ZT_VirtualNetworkRoute*)&(nd->routes)[0], &(nd->num_routes));
+	*/
+}
+
+void _get_network_details(uint64_t nwid, struct zts_network_details *nd)
+{
+	/*
+    _vtaps_lock.lock();
+    __get_network_details_helper(nwid, nd);
+    _vtaps_lock.unlock();
+	*/
+}
+
+void _get_all_network_details(struct zts_network_details *nds, int *num)
+{
+	/*
+    _vtaps_lock.lock();
+    *num = vtapMap.size();
+    int idx = 0;
+    std::map<uint64_t, VirtualTap*>::iterator it;
+    for (it = vtapMap.begin(); it != vtapMap.end(); it++) {
+        _get_network_details(it->first, &nds[idx]);
+        idx++;
+    }
+    _vtaps_lock.unlock();
+	*/
+}
+
+zts_err_t zts_get_network_details(uint64_t nwid, struct zts_network_details *nd)
+{
+	/*
+	_service_lock.lock();
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!nd || nwid == 0) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (!service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	if (retval == ZTS_ERR_OK) {
+		_get_network_details(nwid, nd);
+	}
+	_service_lock.unlock();
+	return retval;
+	*/
+	return 0;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_get_all_network_details(struct zts_network_details *nds, int *num)
+{
+	/*
+	_service_lock.lock();
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!nds || !num) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (!service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	if (retval == ZTS_ERR_OK) {
+		_get_all_network_details(nds, num);
+	}
+	_service_lock.unlock();
+	return retval;	
+	*/
+	return 0;
+}
+#ifdef SDK_JNI
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Multipath/QoS                                                            //
+//////////////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////////////
+// Misc                                                                     //
+//////////////////////////////////////////////////////////////////////////////
+
+int zts_ready()
+{
+	Mutex::Lock _l(_service_lock);
+	return _run_service && _run_lwip_tcpip;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+} // namespace ZeroTier
--- a/src/Controls.hpp
+++ b/src/Controls.hpp
@@ -0,0 +1,91 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Header for ZeroTier service controls
+ */
+
+#ifndef LIBZT_CONTROLS_HPP
+#define LIBZT_CONTROLS_HPP
+
+namespace ZeroTier {
+
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Internal Service Controls                                       //
+//////////////////////////////////////////////////////////////////////////////
+
+void postEvent(uint64_t id, int eventCode);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Starts a ZeroTier service in the background
+ *
+ * @usage For internal use only.
+ * @param
+ * @return
+ */
+#if defined(_WIN32)
+DWORD WINAPI _zts_run_service(LPVOID thread_id);
+#else
+void *_zts_run_service(void *thread_id);
+#endif
+
+/**
+ * @brief [Should not be called from user application] This function must be surrounded by 
+ * ZT service locks. It will determine if it is currently safe and allowed to operate on 
+ * the service.
+ * @usage Can be called at any time
+ * @return 1 or 0
+ */
+int __zts_can_perform_service_operation();
+
+/**
+ * @brief [Should not be called from user application] Returns whether or not the node is 
+ * online.
+ * @usage Can be called at any time
+ * @return 1 or 0
+ */
+int __zts_node_online();
+
+/**
+ * @brief [Should not be called from user application] Adjusts the delay multiplier for the
+ * network stack driver thread.
+ * @usage Can be called at any time
+ */
+void _hibernate_if_needed();
+
+#ifdef __cplusplus
+}
+#endif
+
+} // namespace ZeroTier
+
+#endif // _H
--- a/src/Debug.hpp
+++ b/src/Debug.hpp
@@ -0,0 +1,167 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Debug macros
+ */
+
+#ifndef LIBZT_DEBUG_HPP
+#define LIBZT_DEBUG_HPP
+
+//////////////////////////////////////////////////////////////////////////////
+// Debugging Macros                                                         //
+//////////////////////////////////////////////////////////////////////////////
+
+#if defined(__linux__) || defined(__APPLE__)
+#include <sys/syscall.h>
+#include <pthread.h>
+#include <unistd.h>
+#endif
+#include <string.h>
+
+#define ZT_MSG_ERROR       true // Errors
+#define ZT_MSG_INFO        true // Information which is generally useful to any developer
+#define ZT_MSG_TEST        true // For use in selftest
+#define ZT_MSG_TRANSFER    true // RX/TX specific statements
+
+#define ZT_COLOR           true
+
+// Debug output colors
+#if defined(__APPLE__)
+		#include "TargetConditionals.h"
+#endif
+#if defined(ZT_COLOR) && !defined(_WIN32) && !defined(__ANDROID__) && !defined(TARGET_OS_IPHONE) && !defined(TARGET_IPHONE_SIMULATOR) && !defined(__APP_FRAMEWORK__)
+	#define ZT_RED   "\x1B[31m"
+	#define ZT_GRN   "\x1B[32m"
+	#define ZT_YEL   "\x1B[33m"
+	#define ZT_BLU   "\x1B[34m"
+	#define ZT_MAG   "\x1B[35m"
+	#define ZT_CYN   "\x1B[36m"
+	#define ZT_WHT   "\x1B[37m"
+	#define ZT_RESET "\x1B[0m"
+#else
+	#define ZT_RED
+	#define ZT_GRN
+	#define ZT_YEL
+	#define ZT_BLU
+	#define ZT_MAG
+	#define ZT_CYN
+	#define ZT_WHT
+	#define ZT_RESET
+#endif
+
+#define ZT_FILENAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) // short
+
+#if defined(__JNI_LIB__)
+		#include <jni.h>
+#endif
+#if defined(__ANDROID__)
+		#include <android/log.h>
+		#define ZT_LOG_TAG "ZTSDK"
+#endif
+
+#if defined(LIBZT_DEBUG) || defined(LIBZT_TRACE) || defined(__NATIVETEST__)
+	//
+	#if ZT_MSG_ERROR == true
+		#if defined(__ANDROID__)
+			#define DEBUG_ERROR(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"%17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#elif defined(_WIN32)
+			#define DEBUG_ERROR(fmt, ...) fprintf(stderr, ZT_RED "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+		#else
+		#define DEBUG_ERROR(fmt, args ...) fprintf(stderr, ZT_RED "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_ERROR(fmt, args...)
+	#endif
+
+	//
+	#if ZT_MSG_TEST == true
+		#if defined(__ANDROID__)
+			#define DEBUG_TEST(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"%17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#elif defined(_WIN32)
+			#define DEBUG_TEST(fmt, ...) fprintf(stderr, ZT_CYN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+		#else
+		#define DEBUG_TEST(fmt, args ...) fprintf(stderr, ZT_CYN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_TEST(fmt, args...)
+	#endif
+
+	//
+	#if ZT_MSG_INFO == true
+		#if defined(__ANDROID__)
+			#define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"%17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#elif defined(_WIN32)
+			#define DEBUG_INFO(fmt, ...) fprintf(stderr, ZT_WHT "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+		#else
+			#define DEBUG_INFO(fmt, args ...) fprintf(stderr, ZT_WHT "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_INFO(fmt, args...)
+	#endif
+
+	//
+	#if ZT_MSG_TRANSFER == true
+		#if defined(__ANDROID__)
+			#define DEBUG_TRANS(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"%17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#elif defined(_WIN32)
+			#define DEBUG_TRANS(fmt, ...) fprintf(stderr, ZT_GRN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+		#else
+			#define DEBUG_TRANS(fmt, args ...) fprintf(stderr, ZT_GRN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_TRANS(fmt, args...)
+	#endif
+
+#else // !LIBZT_DEBUG || !__NATIVE_TEST__
+	#if defined(_WIN32)
+		#define DEBUG_ERROR(...)
+		#define DEBUG_TEST(...)
+		#define DEBUG_INFO(...)
+		#define DEBUG_TRANS(...)
+	#else
+		#define DEBUG_ERROR(fmt, args...)
+		#define DEBUG_TEST(fmt, args...)
+		#define DEBUG_INFO(fmt, args...)
+		#define DEBUG_TRANS(fmt, args...)
+	#endif
+#endif
+
+#endif // _H
--- a/src/Intercept.cpp
+++ b/src/Intercept.cpp
--- a/src/Intercept.hpp
+++ b/src/Intercept.hpp
--- a/src/Options.h
+++ b/src/Options.h
@@ -0,0 +1,59 @@
+#ifndef LIBZT_OPTIONS_H
+#define LIBZT_OPTIONS_H
+
+//////////////////////////////////////////////////////////////////////////////
+// Callbacks                                                                //
+//////////////////////////////////////////////////////////////////////////////
+
+#define ZTS_NODE_CALLBACKS    1
+#define ZTS_NETWORK_CALLBACKS 1
+#define ZTS_NETIF_CALLBACKS   1
+#define ZTS_PEER_CALLBACKS    1
+
+/**
+ * The maximum number of un-processed callback messages
+ */
+#define ZTS_CALLBACK_MSG_QUEUE_LEN 256
+
+//////////////////////////////////////////////////////////////////////////////
+// Timing                                                                   //
+//////////////////////////////////////////////////////////////////////////////
+
+/**
+ * How often callback messages are assembled and/or sent
+ */
+#define ZTS_CALLBACK_PROCESSING_INTERVAL 25
+
+/**
+ * Polling interval (in ms) for file descriptors wrapped in the Phy I/O loop (for raw drivers only)
+ */
+#define ZTS_PHY_POLL_INTERVAL 1
+
+#define ZTS_HOUSEKEEPING_INTERVAL 50
+
+/**
+ * By how much thread I/O and callback loop delays are multiplied (unitless)
+ */
+#define ZTS_HIBERNATION_MULTIPLIER 50
+
+//////////////////////////////////////////////////////////////////////////////
+// Thread names                                                             //
+//////////////////////////////////////////////////////////////////////////////
+
+#define ZTS_SERVICE_THREAD_NAME "ZeroTierServiceThread"
+
+#define ZTS_EVENT_CALLBACK_THREAD_NAME "ZeroTierEventCallbackThread"
+
+
+
+
+#define LWIP_FRAMES_HANDLED_PER_CORE_CALL 16 // How many frames are handled per call from core
+#define LWIP_GUARDED_BUF_CHECK_INTERVAL 5 // in ms
+#define LWIP_MAX_GUARDED_RX_BUF_SZ 1024 // number of frame pointers that can be cached waiting for receipt into core
+
+
+
+#define PEER_CACHING 0
+
+
+#endif
--- a/src/RingBuffer.cpp
+++ b/src/RingBuffer.cpp
@@ -1,143 +0,0 @@
-/*
- * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * You can be released from the requirements of the license by purchasing
- * a commercial license. Buying such a license is mandatory as soon as you
- * develop commercial closed-source software that incorporates or links
- * directly against ZeroTier software without disclosing the source code
- * of your own application.
- */
-
-/**
- * @file
- *
- * Ring buffer implementation for network stack drivers
- */
-
-#include <memory.h>
-#include <algorithm>
-
-#include "RingBuffer.h"
-
-bufElementType* RingBuffer::get_buf()
-{
-	return buf + begin;
-}
-
-size_t RingBuffer::produce(size_t n)
-{
-	n = std::min(n, getFree());
-	if (n == 0) {
-		return n;
-	}
-	const size_t first_chunk = std::min(n, size - end);
-	end = (end + first_chunk) % size;
-	if (first_chunk < n) {
-		const size_t second_chunk = n - first_chunk;
-		end = (end + second_chunk) % size;
-	}
-	if (begin == end) {
-		wrap = true;
-	}
-	return n;
-}
-
-void RingBuffer::reset()
-{
-	consume(count());
-}
-
-size_t RingBuffer::consume(size_t n)
-{
-	n = std::min(n, count());
-	if (n == 0) {
-		return n;
-	}
-	if (wrap) {
-		wrap = false;
-	}
-	const size_t first_chunk = std::min(n, size - begin);
-	begin = (begin + first_chunk) % size;
-	if (first_chunk < n) {
-		const size_t second_chunk = n - first_chunk;
-		begin = (begin + second_chunk) % size;
-	}
-	return n;
-}
-
-size_t RingBuffer::write(const bufElementType * data, size_t n)
-{
-	n = std::min(n, getFree());
-
-	if (n == 0) {
-		return n;
-	}
-	const size_t first_chunk = std::min(n, size - end);
-	memcpy(buf + end, data, first_chunk * sizeof(bufElementType));
-	end = (end + first_chunk) % size;
-	if (first_chunk < n) {
-		const size_t second_chunk = n - first_chunk;
-		memcpy(buf + end, data + first_chunk, second_chunk * sizeof(bufElementType));
-		end = (end + second_chunk) % size;
-	}
-	if (begin == end) {
-		wrap = true;
-	}
-	return n;
-}
-
-size_t RingBuffer::read(bufElementType * dest, size_t n)
-{
-	n = std::min(n, count());
-
-	if (n == 0) {
-		return n;
-	}
-	if (wrap) {
-		wrap = false;
-	}
-	const size_t first_chunk = std::min(n, size - begin);
-	memcpy(dest, buf + begin, first_chunk * sizeof(bufElementType));
-	begin = (begin + first_chunk) % size;
-	if (first_chunk < n) {
-		const size_t second_chunk = n - first_chunk;
-		memcpy(dest + first_chunk, buf + begin, second_chunk * sizeof(bufElementType));
-		begin = (begin + second_chunk) % size;
-	}
-	return n;
-}
-
-size_t RingBuffer::count()
-{
-	if (end == begin) {
-		return wrap ? size : 0;
-	}
-	else if (end > begin) {
-		return end - begin;
-	}
-	else {
-		return size + end - begin;
-	}
-}
-
-size_t RingBuffer::getFree()
-{
-	return size - count();
-}
-
--- a/src/Service.cpp
+++ b/src/Service.cpp
--- a/src/Service.hpp
+++ b/src/Service.hpp
@@ -0,0 +1,203 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_ONESERVICE_HPP
+#define ZT_ONESERVICE_HPP
+
+#include <string>
+#include <vector>
+
+#ifdef SDK_JNI
+#include <jni.h>
+#endif
+
+namespace ZeroTier {
+
+class VirtualTap;
+// Use the virtual libzt endpoint instead of a tun/tap port driver
+namespace ZeroTier { typedef VirtualTap EthernetTap; }
+
+// Forward declaration so we can avoid dragging everything in
+struct InetAddress;
+class Node;
+
+/**
+ * Local service for ZeroTier One as system VPN/NFV provider
+ */
+class OneService
+{
+public:
+
+	/**
+	 * Returned by node main if/when it terminates
+	 */
+	enum ReasonForTermination
+	{
+		/**
+		 * Instance is still running
+		 */
+		ONE_STILL_RUNNING = 0,
+
+		/**
+		 * Normal shutdown
+		 */
+		ONE_NORMAL_TERMINATION = 1,
+
+		/**
+		 * A serious unrecoverable error has occurred
+		 */
+		ONE_UNRECOVERABLE_ERROR = 2,
+
+		/**
+		 * Your identity has collided with another
+		 */
+		ONE_IDENTITY_COLLISION = 3
+	};
+
+	/**
+	 * Local settings for each network
+	 */
+	struct NetworkSettings
+	{
+		/**
+		 * Allow this network to configure IP addresses and routes?
+		 */
+		bool allowManaged;
+
+		/**
+		 * Whitelist of addresses that can be configured by this network.
+		 * If empty and allowManaged is true, allow all private/pseudoprivate addresses.
+		 */
+		std::vector<InetAddress> allowManagedWhitelist;
+
+		/**
+		 * Allow configuration of IPs and routes within global (Internet) IP space?
+		 */
+		bool allowGlobal;
+
+		/**
+		 * Allow overriding of system default routes for "full tunnel" operation?
+		 */
+		bool allowDefault;
+	};
+
+	/**
+	 * @return Platform default home path or empty string if this platform doesn't have one
+	 */
+	static std::string platformDefaultHomePath();
+
+	/**
+	 * Create a new instance of the service
+	 *
+	 * Once created, you must call the run() method to actually start
+	 * processing.
+	 *
+	 * The port is saved to a file in the home path called zerotier-one.port,
+	 * which is used by the CLI and can be used to see which port was chosen if
+	 * 0 (random port) is picked.
+	 *
+	 * @param hp Home path
+	 * @param port TCP and UDP port for packets and HTTP control (if 0, pick random port)
+	 */
+	static OneService *newInstance(const char *hp,unsigned int port);
+
+	virtual ~OneService();
+
+	/**
+	 * Execute the service main I/O loop until terminated
+	 *
+	 * The terminate() method may be called from a signal handler or another
+	 * thread to terminate execution. Otherwise this will not return unless
+	 * another condition terminates execution such as a fatal error.
+	 */
+	virtual ReasonForTermination run() = 0;
+
+	/**
+	 * @return Reason for terminating or ONE_STILL_RUNNING if running
+	 */
+	virtual ReasonForTermination reasonForTermination() const = 0;
+
+	/**
+	 * @return Fatal error message or empty string if none
+	 */
+	virtual std::string fatalErrorMessage() const = 0;
+
+	/**
+	 * @return System device name corresponding with a given ZeroTier network ID or empty string if not opened yet or network ID not found
+	 */
+	virtual std::string portDeviceName(uint64_t nwid) const = 0;
+
+	/**
+	 * Whether we allow access to the service via local HTTP requests (disabled by default in libzt)
+	 */
+	bool allowHttpBackplaneManagement = false;
+
+	/**
+	 * @return Reference to the Node
+	 */
+	virtual Node * getNode() = 0;
+
+	/**
+	 * Fills out a structure with network-specific route information
+	 */
+	virtual void getRoutes(uint64_t nwid, void *routeArray, unsigned int *numRoutes) = 0;
+
+	virtual int networkCount() = 0;
+	virtual void leaveAll() = 0;
+	virtual void join(uint64_t nwid) = 0;
+	virtual void leave(uint64_t nwid) = 0;
+	virtual int getPeerStatus(uint64_t id) = 0;
+	
+	/**
+	 * Terminate background service (can be called from other threads)
+	 */
+	virtual void terminate() = 0;
+
+	/**
+	 * Get local settings for a network
+	 *
+	 * @param nwid Network ID
+	 * @param settings Buffer to fill with local network settings
+	 * @return True if network was found and settings is filled
+	 */
+	virtual bool getNetworkSettings(const uint64_t nwid,NetworkSettings &settings) const = 0;
+
+	/**
+	 * @return True if service is still running
+	 */
+	inline bool isRunning() const { return (this->reasonForTermination() == ONE_STILL_RUNNING); }
+
+protected:
+	OneService() {}
+
+private:
+	OneService(const OneService &one) {}
+	inline OneService &operator=(const OneService &one) { return *this; }
+};
+
+} // namespace ZeroTier
+
+#endif
--- a/src/ServiceControls.cpp
+++ b/src/ServiceControls.cpp
--- a/src/Sockets.cpp
+++ b/src/Sockets.cpp
@@ -33,9 +33,12 @@
 #include <string.h>

 #include "lwip/sockets.h"
-#include "Defs.hpp"
+#include "lwip/def.h"
+
 #include "libzt.h"
+#include "Options.h"
 #include "Debug.hpp"
+#include "Controls.hpp"

 #ifdef SDK_JNI
 	#include <jni.h>
@@ -45,67 +48,27 @@
 	//#include <sys/select.h>
 	//#include <sys/ioctl.h>
 #endif
+#endif

-#ifdef __cplusplus
-extern "C" {
-
-#endif
-	void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
-	void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
-	void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set);
-	void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set);
-#ifdef __cplusplus
-}
-#endif
-#endif
+namespace ZeroTier {

 //////////////////////////////////////////////////////////////////////////////
 // Socket API                                                               //
 //////////////////////////////////////////////////////////////////////////////

+extern bool _run_service;
+extern bool _run_lwip_tcpip;
+
 #ifdef __cplusplus
 extern "C" {
 #endif

-// Custom errno to prevent conflicts with platform's own errno
-int zts_errno;
-
-// lwIP prototypes copied from lwip/src/include/sockets.h
-// Don't call these directly, call zts_* functions instead
-int lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
-int lwip_bind(int s, const struct sockaddr *name, socklen_t namelen);
-int lwip_shutdown(int s, int how);
-int lwip_getpeername (int s, struct sockaddr *name, socklen_t *namelen);
-int lwip_getsockname (int s, struct sockaddr *name, socklen_t *namelen);
-int lwip_getsockopt (int s, int level, int optname, void *optval, socklen_t *optlen);
-int lwip_setsockopt (int s, int level, int optname, const void *optval, socklen_t optlen);
- int lwip_close(int s);
-int lwip_connect(int s, const struct sockaddr *name, socklen_t namelen);
-int lwip_listen(int s, int backlog);
-ssize_t lwip_recv(int s, void *mem, size_t len, int flags);
-ssize_t lwip_read(int s, void *mem, size_t len);
-ssize_t lwip_readv(int s, const struct iovec *iov, int iovcnt);
-ssize_t lwip_recvfrom(int s, void *mem, size_t len, int flags,
-      struct sockaddr *from, socklen_t *fromlen);
-ssize_t lwip_recvmsg(int s, struct msghdr *message, int flags);
-ssize_t lwip_send(int s, const void *dataptr, size_t size, int flags);
-ssize_t lwip_sendmsg(int s, const struct msghdr *message, int flags);
-ssize_t lwip_sendto(int s, const void *dataptr, size_t size, int flags,
-    const struct sockaddr *to, socklen_t tolen);
-int lwip_socket(int domain, int type, int protocol);
-ssize_t lwip_write(int s, const void *dataptr, size_t size);
-ssize_t lwip_writev(int s, const struct iovec *iov, int iovcnt);
-#if LWIP_SOCKET_SELECT
-int lwip_select(int maxfdp1, fd_set *readset, fd_set *writeset, fd_set *exceptset,
-                struct timeval *timeout);
+#ifdef SDK_JNI
+void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set);
+void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set);
 #endif
-#if LWIP_SOCKET_POLL
-int lwip_poll(struct pollfd *fds, nfds_t nfds, int timeout);
-#endif
-int lwip_ioctl(int s, long cmd, void *argp);
-int lwip_fcntl(int s, int cmd, int val);
-const char *lwip_inet_ntop(int af, const void *src, char *dst, socklen_t size);
-int lwip_inet_pton(int af, const char *src, void *dst);

 // Copied from lwip/src/include/sockets.h and renamed to prevent a name collision
 // with system definitions
@@ -119,11 +82,11 @@ struct lwip_sockaddr {
 // ZeroTier Socket API                                                      //
 //////////////////////////////////////////////////////////////////////////////

-int zts_ready();
+extern int zts_errno;

 int zts_socket(int socket_family, int socket_type, int protocol)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_socket(socket_family, socket_type, protocol);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_socket(socket_family, socket_type, protocol);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
@@ -142,7 +105,7 @@ int zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen)
 	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_connect(fd, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_connect(fd, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_connect(
@@ -164,7 +127,7 @@ int zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen)
 	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_bind(fd, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_bind(fd, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_bind(
@@ -180,7 +143,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_bind(

 int zts_listen(int fd, int backlog)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_listen(fd, backlog);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_listen(fd, backlog);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(
@@ -193,7 +156,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(

 int zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_accept(fd, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_accept(fd, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept(
@@ -210,7 +173,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept(
 #if defined(__linux__)
 int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)
 {
-	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP;
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP;
 }
 #endif
 #ifdef SDK_JNI
@@ -229,7 +192,7 @@ int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)

 int zts_setsockopt(int fd, int level, int optname, const void *optval, socklen_t optlen)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_setsockopt(fd, level, optname, optval, optlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_setsockopt(fd, level, optname, optval, optlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
@@ -279,7 +242,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(

 int zts_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getsockopt(fd, level, optname, optval, optlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getsockopt(fd, level, optname, optval, optlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
@@ -340,7 +303,7 @@ int zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen)
 	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getsockname(fd, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getsockname(fd, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_getsockname(JNIEnv *env, jobject thisObj,
@@ -362,7 +325,7 @@ int zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen)
 	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getpeername(fd, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getpeername(fd, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getpeername(JNIEnv *env, jobject thisObj,
@@ -377,23 +340,23 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getpeername(JNIEnv *env,

 int zts_gethostname(char *name, size_t len)
 {
-	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
 }
 #ifdef SDK_JNI
 #endif

 int zts_sethostname(const char *name, size_t len)
 {
-	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
 }
 #ifdef SDK_JNI
 #endif

+/*
 struct hostent *zts_gethostbyname(const char *name)
 {
-	return (struct hostent *)(!zts_ready() ? NULL : NULL);
+	return (struct hostent *)((!_run_service || !_run_lwip_tcpip) ? NULL : NULL);
 	// TODO: Test thread safety
-	/*
 	char buf[256];
 	int buflen = 256;
 	int h_err = 0;
@@ -409,14 +372,14 @@ struct hostent *zts_gethostbyname(const char *name)
 	return *result;
 	
 	return lwip_gethostbyname(name);
-	*/
 }
 #ifdef SDK_JNI
 #endif
+*/

 int zts_close(int fd)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_close(fd);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_close(fd);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_close(
@@ -429,7 +392,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_close(
 int zts_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 	struct timeval *timeout)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_select(nfds, readfds, writefds, exceptfds, timeout);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_select(nfds, readfds, writefds, exceptfds, timeout);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_select(JNIEnv *env, jobject thisObj,
@@ -482,7 +445,7 @@ int zts_fcntl(int fd, int cmd, int flags)
 		translated_flags = 1;
 	}
 #endif
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_fcntl(fd, cmd, translated_flags);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_fcntl(fd, cmd, translated_flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_fcntl(
@@ -498,7 +461,7 @@ int zts_ioctl(int fd, unsigned long request, void *argp)
 	if (!argp) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_ioctl(fd, request, argp);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_ioctl(fd, request, argp);
 }
 #ifdef SDK_JNI
 JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
@@ -532,7 +495,7 @@ ssize_t zts_send(int fd, const void *buf, size_t len, int flags)
 	if (!buf || len <= 0) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_send(fd, buf, len, flags);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_send(fd, buf, len, flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_send(
@@ -554,7 +517,7 @@ ssize_t zts_sendto(int fd, const void *buf, size_t len, int flags,
 	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_sendto(fd, buf, len, flags, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_sendto(fd, buf, len, flags, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(
@@ -572,7 +535,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(

 ssize_t zts_sendmsg(int fd, const struct msghdr *msg, int flags)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_sendmsg(fd, msg, flags);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_sendmsg(fd, msg, flags);
 }
 #ifdef SDK_JNI
 #endif
@@ -582,7 +545,7 @@ ssize_t zts_recv(int fd, void *buf, size_t len, int flags)
 	if (!buf) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_recv(fd, buf, len, flags);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_recv(fd, buf, len, flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recv(JNIEnv *env, jobject thisObj,
@@ -601,7 +564,7 @@ ssize_t zts_recvfrom(int fd, void *buf, size_t len, int flags,
 	if (!buf) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_recvfrom(fd, buf, len, flags, addr, addrlen);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_recvfrom(fd, buf, len, flags, addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recvfrom(
@@ -619,7 +582,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recvfrom(

 ssize_t zts_recvmsg(int fd, struct msghdr *msg, int flags)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // Not currently implemented by stack
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : -1; // Not currently implemented by stack
 }
 #ifdef SDK_JNI
 #endif
@@ -629,7 +592,7 @@ int zts_read(int fd, void *buf, size_t len)
 	if (!buf) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_read(fd, buf, len);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_read(fd, buf, len);
 }
 int zts_read_offset(int fd, void *buf, size_t offset, size_t len)
 {
@@ -637,7 +600,7 @@ int zts_read_offset(int fd, void *buf, size_t offset, size_t len)
 		return ZTS_ERR_INVALID_ARG;
 	}
 	char *cbuf = (char*)buf;
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_read(fd, &(cbuf[offset]), len);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_read(fd, &(cbuf[offset]), len);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read(JNIEnv *env, jobject thisObj,
@@ -671,7 +634,7 @@ int zts_write(int fd, const void *buf, size_t len)
 	if (!buf || len <= 0) {
 		return ZTS_ERR_INVALID_ARG;
 	}
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_write(fd, buf, len);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_write(fd, buf, len);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write__IB(JNIEnv *env, jobject thisObj,
@@ -700,7 +663,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1byte(JNIEnv *env,

 int zts_shutdown(int fd, int how)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_shutdown(fd, how);
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_shutdown(fd, how);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_shutdown(
@@ -712,14 +675,14 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_shutdown(

 int zts_add_dns_nameserver(struct sockaddr *addr)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // TODO
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : -1; // TODO
 }
 #ifdef SDK_JNI
 #endif

 int zts_del_dns_nameserver(struct sockaddr *addr)
 {
-	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // TODO
+	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : -1; // TODO
 }
 #ifdef SDK_JNI
 #endif
@@ -778,7 +741,7 @@ void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	{
 		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
 		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
-		(*env).SetIntField(addr, fid, ntohs(in4->sin_port));
+		(*env).SetIntField(addr, fid, lwip_ntohs(in4->sin_port));
 		fid = (*env).GetFieldID(c,"_family", "I");
 		(*env).SetIntField(addr, fid, (in4->sin_family));
 		fid = env->GetFieldID(c, "_ip4", "[B");
@@ -794,7 +757,7 @@ void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	{
 		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
 		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
-		(*env).SetIntField(addr, fid, ntohs(in6->sin6_port));
+		(*env).SetIntField(addr, fid, lwip_ntohs(in6->sin6_port));
 		fid = (*env).GetFieldID(c,"_family", "I");
 		(*env).SetIntField(addr, fid, (in6->sin6_family));
 		fid = env->GetFieldID(c, "_ip6", "[B");
@@ -819,7 +782,7 @@ void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	{
 		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
 		fid = (*env).GetFieldID(c, "_port", "I");
-		in4->sin_port = htons((*env).GetIntField(addr, fid));
+		in4->sin_port = lwip_htons((*env).GetIntField(addr, fid));
 		in4->sin_family = AF_INET;
 		fid = env->GetFieldID(c, "_ip4", "[B");
 		jobject ipData = (*env).GetObjectField (addr, fid);
@@ -833,7 +796,7 @@ void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	{
 		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
 		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
-		in6->sin6_port = htons((*env).GetIntField(addr, fid));
+		in6->sin6_port = lwip_htons((*env).GetIntField(addr, fid));
 		fid = (*env).GetFieldID(c,"_family", "I");
 		in6->sin6_family = AF_INET6;
 		fid = env->GetFieldID(c, "_ip6", "[B");
@@ -850,3 +813,5 @@ void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 #ifdef __cplusplus
 }
 #endif
+
+} // namespace ZeroTier
--- a/src/VirtualTap.cpp
+++ b/src/VirtualTap.cpp
@@ -35,13 +35,12 @@
 #include "Node.hpp"
 #include "OSUtils.hpp"

-#include "Constants.hpp" // libzt
-#include "ServiceControls.hpp"
-#include "OneService.hpp"
+#include "Service.hpp"
 extern void _push_callback_event(uint64_t nwid, int eventCode);

 #include "Mutex.hpp"
-#include "lwIP.h"
+#include "lwipDriver.hpp"
+#include "libzt.h"

 #ifdef _MSC_VER
 #include "Synchapi.h"
@@ -51,10 +50,7 @@ namespace ZeroTier {

 class VirtualTap;

-extern OneService *zt1Service;
-extern void (*_userCallbackFunc)(uint64_t, int);
-extern std::map<uint64_t, VirtualTap*> vtapMap;
-extern Mutex _vtaps_lock;
+extern OneService *service;

 /**
 * A virtual tap device. The ZeroTier core service creates one of these for each
@@ -86,11 +82,6 @@ VirtualTap::VirtualTap(
 	snprintf(vtap_full_name, sizeof(vtap_full_name), "libzt%llx", (unsigned long long)_nwid);
 	_dev = vtap_full_name;
 	::pipe(_shutdownSignalPipe);
-
-	_vtaps_lock.lock();
-	vtapMap[_nwid] = this;
-	_vtaps_lock.unlock();
-
 	lwip_driver_init();
 	// Start virtual tap thread and stack I/O loops
 	_thread = Thread::start(this);
@@ -198,10 +189,10 @@ std::string VirtualTap::deviceName() const

 std::string VirtualTap::nodeId() const
 {
-	if (zt1Service) {
-		char id[ZTS_ID_LEN];
+	if (service) {
+		char id[16];
 		memset(id, 0, sizeof(id));
-		sprintf(id, "%llx", (unsigned long long)((OneService *)zt1Service)->getNode()->address());
+		sprintf(id, "%llx", (unsigned long long)((OneService *)service)->getNode()->address());
 		return std::string(id);
 	}
 	else {
@@ -283,74 +274,9 @@ void VirtualTap::threadMain()

 void VirtualTap::Housekeeping()
 {
-/*
-	Mutex::Lock _l(_tap_m);
-	OneService *service = ((OneService *)zt1Service);
-	if (!service) {
-		return;
-	}
-	nd.num_routes = ZTS_MAX_NETWORK_ROUTES;
-	service->getRoutes(this->_nwid, (ZT_VirtualNetworkRoute*)&(nd.routes)[0], &(nd.num_routes));
-*/
-
-/*			
-	InetAddress target_addr;
-	InetAddress via_addr;
-	InetAddress null_addr;
-	InetAddress nm;
-	null_addr.fromString("");
-	bool found;
-	char ipbuf[INET6_ADDRSTRLEN], ipbuf2[INET6_ADDRSTRLEN], ipbuf3[INET6_ADDRSTRLEN];
-
-	// TODO: Rework this when we have time
-	// check if pushed route exists in tap (add)
-	/*
-	for (int i=0; i<ZT_MAX_NETWORK_ROUTES; i++) {
-		found = false;
-		target_addr = managed_routes->at(i).target;
-		via_addr = managed_routes->at(i).via;
-		nm = target_addr.netmask();
-		for (size_t j=0; j<routes.size(); j++) {
-			if (via_addr.ipsEqual(null_addr) || target_addr.ipsEqual(null_addr)) {
-				found=true;
-				continue;
-			}
-			if (routes[j].first.ipsEqual(target_addr) && routes[j].second.ipsEqual(nm)) {
-				found=true;
-			}
-		}
-		if (found == false) {
-			if (via_addr.ipsEqual(null_addr) == false) {
-				DEBUG_INFO("adding route <target=%s, nm=%s, via=%s>", target_addr.toString(ipbuf), nm.toString(ipbuf2), via_addr.toString(ipbuf3));
-				routes.push_back(std::pair<InetAddress,InetAddress>(target_addr, nm));
-				routeAdd(target_addr, nm, via_addr);
-			}
-		}
-	}
-	// check if route exists in tap but not in pushed routes (remove)
-	for (size_t i=0; i<routes.size(); i++) {
-		found = false;
-		for (int j=0; j<ZT_MAX_NETWORK_ROUTES; j++) {
-			target_addr = managed_routes->at(j).target;
-			via_addr = managed_routes->at(j).via;
-			nm = target_addr.netmask();
-			if (routes[i].first.ipsEqual(target_addr) && routes[i].second.ipsEqual(nm)) {
-				found=true;
-			}
-		}
-		if (found == false) {
-			DEBUG_INFO("removing route to <%s,%s>", routes[i].first.toString(ipbuf), routes[i].second.toString(ipbuf2));
-			routes.erase(routes.begin() + i);
-			routeDelete(routes[i].first, routes[i].second);
-		}
-	}
-*/
+    //
 }

-//////////////////////////////////////////////////////////////////////////////
-// Not used in this implementation                                          //
-//////////////////////////////////////////////////////////////////////////////
-
 void VirtualTap::phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *local_address,
 	const struct sockaddr *from,void *data,unsigned long len) {}
 void VirtualTap::phyOnTcpConnect(PhySocket *sock,void **uptr,bool success) {}
--- a/src/VirtualTap.hpp
+++ b/src/VirtualTap.hpp
@@ -0,0 +1,263 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Virtual Ethernet tap device
+ */
+
+#ifndef LIBZT_VIRTUALTAP_HPP
+#define LIBZT_VIRTUALTAP_HPP
+
+#ifndef _MSC_VER
+extern int errno;
+#endif
+
+#include "Phy.hpp"
+#include "Thread.hpp"
+#include "InetAddress.hpp"
+#include "MulticastGroup.hpp"
+#include "Mutex.hpp"
+
+#include "Options.h"
+
+#if defined(_WIN32)
+#include <WinSock2.h>
+#include <Windows.h>
+#include <IPHlpApi.h>
+#include <Ifdef.h>
+#endif
+
+namespace ZeroTier {
+
+class Mutex;
+
+/**
+ * A virtual tap device. The ZeroTier core service creates one of these for each
+ * virtual network joined. It will be destroyed upon leave().
+ */
+class VirtualTap
+{
+	friend class Phy<VirtualTap *>;
+
+public:
+	VirtualTap(
+		const char *homePath,
+		const MAC &mac,
+		unsigned int mtu,
+		unsigned int metric,
+		uint64_t nwid,
+		const char *friendlyName,
+		void (*handler)(void *, void *, uint64_t, const MAC &,
+			const MAC &, unsigned int, unsigned int, const void *, unsigned int),
+		void *arg);
+
+	~VirtualTap();
+
+	void setEnabled(bool en);
+	bool enabled() const;
+
+	/**
+	 * Adds an address to the userspace stack interface associated with this VirtualTap
+	 * - Starts VirtualTap main thread ONLY if successful
+	 */
+	bool addIp(const InetAddress &ip);
+
+	/**
+	 * Removes an address from the userspace stack interface associated with this VirtualTap
+	 */
+	bool removeIp(const InetAddress &ip);
+
+	/**
+	 * Presents data to the userspace stack
+	 */
+	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,
+		unsigned int len);
+
+	/**
+	 * Get VirtualTap device name (e.g. 'libzt17d72843bc2c5760')
+	 */
+	std::string deviceName() const;
+
+	/**
+	 * Get Node ID (ZT address)
+	 */
+	std::string nodeId() const;
+
+	/**
+	 * Set friendly name
+	 */
+	void setFriendlyName(const char *friendlyName);
+
+	/**
+	 * Scan multicast groups
+	 */
+	void scanMulticastGroups(std::vector<MulticastGroup> &added,
+		std::vector<MulticastGroup> &removed);
+
+	/**
+	 * Set MTU
+	 */
+	void setMtu(unsigned int mtu);
+
+	/**
+	 * Calls main network stack loops
+	 */
+	void threadMain()
+		throw();
+
+#if defined(__MINGW32__)
+	/* The following is merely to make ZeroTier's OneService happy while building on Windows.
+		we won't use these in libzt */
+	NET_LUID _deviceLuid;
+	std::string _deviceInstanceId;
+
+	/**
+	 * Returns whether the VirtualTap interface has been initialized
+	 */
+	bool isInitialized() const { return _initialized; };
+
+	inline const NET_LUID &luid() const { return _deviceLuid; }
+	inline const std::string &instanceId() const { return _deviceInstanceId; }
+#endif
+	/**
+	 * For moving data onto the ZeroTier virtual wire
+	 */
+	void (*_handler)(void *, void *, uint64_t, const MAC &, const MAC &, unsigned int, unsigned int,
+		const void *, unsigned int);
+
+	void phyOnUnixClose(PhySocket *sock, void **uptr);
+	void phyOnUnixData(PhySocket *sock, void **uptr, void *data, ssize_t len);
+	void phyOnUnixWritable(PhySocket *sock, void **uptr, bool stack_invoked);
+
+	//////////////////////////////////////////////////////////////////////////////
+	// Lower-level lwIP netif handling and traffic handling readiness           //
+	//////////////////////////////////////////////////////////////////////////////
+
+	void *netif4 = NULL;
+	void *netif6 = NULL;
+
+	bool netif4WasUpLastCheck = false;
+	bool netif6WasUpLastCheck = false;
+
+	/**
+	 * Notes the current state of the lower level lwIP netif and reports if a state change
+	 * has happened since the last check. This method is likely temporary.
+	 */
+	uint64_t recognizeLowerLevelInterfaceStateChange(void *n);
+
+	/**
+	 * A state will only be reported via callback if it differs from this value. Subsequently this
+	 * value will be updated.
+	 */
+	//int _lastReportedStatus;
+
+	/**
+	 * The last time that this virtual tap received a network config update from the core
+	 */
+	uint64_t _lastConfigUpdateTime = 0;
+
+	/**
+	 * The last time that a callback notification was sent to the user application signalling
+	 * that this interface is ready to process traffic.
+	 */
+	uint64_t _lastReadyReportTime = 0;
+
+	void lastConfigUpdate(uint64_t lastConfigUpdateTime);
+
+	int _networkStatus = 0;
+	int _netifStatus = 0;
+	/**
+	 * Returns whether or not this interface is ready for traffic.
+	 */
+	bool isReady();
+
+	//////////////////////////////////////////////////////////////////////////////
+	// Vars                                                                     //
+	//////////////////////////////////////////////////////////////////////////////
+
+	std::vector<std::pair<InetAddress, InetAddress> > routes;
+
+	char vtap_full_name[64];
+	char vtap_abbr_name[16];
+
+	size_t ifindex = 0;
+
+	std::vector<InetAddress> ips() const;
+	std::vector<InetAddress> _ips;
+
+	std::string _homePath;
+	void *_arg;
+	volatile bool _initialized;
+	volatile bool _enabled;
+	volatile bool _run;
+	MAC _mac;
+	unsigned int _mtu;
+	uint64_t _nwid;
+	PhySocket *_unixListenSocket;
+	Phy<VirtualTap *> _phy;
+
+	Thread _thread;
+
+	int _shutdownSignalPipe[2];
+
+	std::string _dev; // path to Unix domain socket
+
+	std::vector<MulticastGroup> _multicastGroups;
+	Mutex _multicastGroups_m;
+	Mutex _ips_m;
+
+	//struct zts_network_details nd;
+
+	/*
+	 * Timestamp of last run of housekeeping
+	 * SEE: ZT_HOUSEKEEPING_INTERVAL in libzt.h
+	 */
+	uint64_t last_housekeeping_ts = 0;
+
+	/**
+	 * Performs miscellaneous background tasks
+	 */
+	void Housekeeping();
+
+	//////////////////////////////////////////////////////////////////////////////
+	// Not used in this implementation                                          //
+	//////////////////////////////////////////////////////////////////////////////
+
+	void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *local_address,
+		const struct sockaddr *from,void *data,unsigned long len);
+	void phyOnTcpConnect(PhySocket *sock,void **uptr,bool success);
+	void phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN,
+		const struct sockaddr *from);
+	void phyOnTcpClose(PhySocket *sock,void **uptr);
+	void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len);
+	void phyOnTcpWritable(PhySocket *sock,void **uptr);
+};
+
+} // namespace ZeroTier
+
+#endif // _H
--- a/src/lwipDriver.cpp
+++ b/src/lwipDriver.cpp
@@ -30,15 +30,10 @@
 * lwIP network stack driver
 */

-#include <vector>
 #include <queue>

 #include "MAC.hpp"
-
 #include "Mutex.hpp"
-#include "Constants.hpp"
-#include "VirtualTap.hpp"
-#include "Constants.hpp" // libzt

 #include "netif/ethernet.h"
 #include "lwip/netif.h"
@@ -53,15 +48,23 @@
 #include "lwip/ethip6.h"
 #include "lwip/ip_addr.h"
 #include "lwip/nd6.h"
-#include "lwip/dns.h"
 #include "lwip/netifapi.h"

-#include "lwIP.h"
+#include "VirtualTap.hpp"
+#include "lwipDriver.hpp"
+#include "libzt.h"
+#include "Controls.hpp"
+extern void postEvent(uint64_t id, int eventCode);

 #if defined(_WIN32)
 #include <time.h>
 #endif

+/**
+ * Length of human-readable MAC address string
+ */
+#define ZTS_MAC_ADDRSTRLEN 18
+
 namespace ZeroTier {

 bool main_loop_exited = false;
@@ -69,13 +72,12 @@ bool lwip_driver_initialized = false;
 bool has_already_been_initialized = false;
 int hibernationDelayMultiplier = 1;

+extern bool _run_lwip_tcpip;
+
 Mutex driver_m;

 std::queue<struct zts_sorted_packet*> rx_queue;
-ZeroTier::Mutex _rx_input_lock_m;
-
-extern void _push_callback_event(uint64_t nwid, int eventCode);
-extern void _process_callback_event_helper(uint64_t nwid, int eventCode);
+Mutex _rx_input_lock_m;

 void lwip_hibernate_driver()
 {
@@ -93,6 +95,8 @@ static void tcpip_init_done(void *arg)
 	sys_sem_t *sem;
 	sem = (sys_sem_t *)arg;
 	lwip_driver_initialized = true;
+	_run_lwip_tcpip = true;
+	postEvent((uint64_t)0, ZTS_EVENT_NETWORK_STACK_UP);
 	driver_m.unlock();
 	sys_sem_signal(sem);
 }
@@ -157,6 +161,7 @@ static void main_lwip_driver_loop(void *arg)
 		tcpip_callback_with_block(my_tcpip_callback, NULL, 1);
 	}
 	main_loop_exited = true;
+	_run_lwip_tcpip = false;
 }

 // Initialize the lwIP stack
@@ -191,14 +196,16 @@ void lwip_driver_shutdown()
 	while(!main_loop_exited) {
 		usleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*1000);
 	}
+	/*
 	if (tcpip_shutdown() == ERR_OK) {
 		sys_timeouts_free();
 	}
+	*/
 }

 void lwip_dispose_of_netifs(void *tapref)
 {
-	ZeroTier::VirtualTap *vtap = (ZeroTier::VirtualTap*)tapref;
+	VirtualTap *vtap = (VirtualTap*)tapref;
 	if (vtap->netif4) {
 		netif_remove((struct netif*)(vtap->netif4));
 		netif_set_down((struct netif*)(vtap->netif4));
@@ -222,7 +229,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 	char *bufptr;
 	int totalLength = 0;

-	ZeroTier::VirtualTap *tap = (ZeroTier::VirtualTap*)netif->state;
+	VirtualTap *tap = (VirtualTap*)netif->state;
 	bufptr = buf;
 	for (q = p; q != NULL; q = q->next) {
 		memcpy(bufptr, q->payload, q->len);
@@ -232,56 +239,54 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 	struct eth_hdr *ethhdr;
 	ethhdr = (struct eth_hdr *)buf;

-	ZeroTier::MAC src_mac;
-	ZeroTier::MAC dest_mac;
+	MAC src_mac;
+	MAC dest_mac;
 	src_mac.setTo(ethhdr->src.addr, 6);
 	dest_mac.setTo(ethhdr->dest.addr, 6);

 	char *data = buf + sizeof(struct eth_hdr);
 	int len = totalLength - sizeof(struct eth_hdr);
-	int proto = ZeroTier::Utils::ntoh((uint16_t)ethhdr->type);
+	int proto = Utils::ntoh((uint16_t)ethhdr->type);
 	tap->_handler(tap->_arg, NULL, tap->_nwid, src_mac, dest_mac, proto, 0, data, len);
 /*
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
-		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[ZTS_ID_LEN];
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
 		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
 			ethhdr->dest.addr[0], ethhdr->dest.addr[1], ethhdr->dest.addr[2], 
 			ethhdr->dest.addr[3], ethhdr->dest.addr[4], ethhdr->dest.addr[5]);
-		ZeroTier::MAC mac;
+		MAC mac;
 		mac.setTo(ethhdr->dest.addr, 6);
 		mac.toAddress(tap->_nwid).toString(nodeBuf);
 		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
-			ZeroTier::Utils::ntoh(ethhdr->type), flagbuf);
+			Utils::ntoh(ethhdr->type), flagbuf);
 	}
 */
-	
-	
 	return ERR_OK;
 }

-void lwip_eth_rx(ZeroTier::VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
+void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
 	const void *data, unsigned int len)
 {
 	struct pbuf *p,*q;
 	struct eth_hdr ethhdr;
 	from.copyTo(ethhdr.src.addr, 6);
 	to.copyTo(ethhdr.dest.addr, 6);
-	ethhdr.type = ZeroTier::Utils::hton((uint16_t)etherType);
+	ethhdr.type = Utils::hton((uint16_t)etherType);
 /*
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
-		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[ZTS_ID_LEN];
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
 		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
 			ethhdr.dest.addr[0], ethhdr.dest.addr[1], ethhdr.dest.addr[2],
 			ethhdr.dest.addr[3], ethhdr.dest.addr[4], ethhdr.dest.addr[5]);
-		ZeroTier::MAC mac;
+		MAC mac;
 		mac.setTo(ethhdr.src.addr, 6);
 		mac.toAddress(tap->_nwid).toString(nodeBuf);
 		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
-			ZeroTier::Utils::ntoh(ethhdr.type), flagbuf);
+			Utils::ntoh(ethhdr.type), flagbuf);
 	}
 */
 	if (etherType == 0x0800 || etherType == 0x0806) { // ip4 or ARP
@@ -390,23 +395,23 @@ static void netif_status_callback(struct netif *netif)
 	if (!netif->state) {
 		return;
 	}
-	ZeroTier::VirtualTap *tap = (ZeroTier::VirtualTap *)netif->state;
+	VirtualTap *tap = (VirtualTap *)netif->state;
 	if (netif->flags & NETIF_FLAG_UP) {
-		ZeroTier::VirtualTap *vtap = (ZeroTier::VirtualTap*)netif->state;
+		VirtualTap *vtap = (VirtualTap*)netif->state;
 		if (netif == vtap->netif6) {
 			// DEBUG_INFO("netif=%p, vtap->netif6=%p", netif, vtap->netif6);
-			_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_UP_IP6);
+			postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP6);
 		}
 		if (netif == vtap->netif4) {
 			// DEBUG_INFO("netif=%p, vtap->netif4=%p", netif, vtap->netif4);
-			_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_UP_IP4);
+			postEvent(tap->_nwid, ZTS_EVENT_NETIF_UP_IP4);
 		}
 	}
 	if (!(netif->flags & NETIF_FLAG_UP)) {
 		if (netif->flags & NETIF_FLAG_MLD6) {
-			_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP6);
+			postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP6);
 		} else {
-			_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP4);
+			postEvent(tap->_nwid, ZTS_EVENT_NETIF_DOWN_IP4);
 		}
 	}
 	// TODO: ZTS_EVENT_NETIF_NEW_ADDRESS
@@ -421,8 +426,8 @@ static void netif_remove_callback(struct netif *netif)
 	if (!netif->state) {
 		return;
 	}
-	ZeroTier::VirtualTap *tap = (ZeroTier::VirtualTap *)netif->state;
-	_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_REMOVED);
+	VirtualTap *tap = (VirtualTap *)netif->state;
+	postEvent(tap->_nwid, ZTS_EVENT_NETIF_REMOVED);
 	//print_netif_info(netif);
 }

@@ -434,12 +439,12 @@ static void netif_link_callback(struct netif *netif)
 	if (!netif->state) {
 		return;
 	}
-	ZeroTier::VirtualTap *tap = (ZeroTier::VirtualTap *)netif->state;
+	VirtualTap *tap = (VirtualTap *)netif->state;
 	if (netif->flags & NETIF_FLAG_LINK_UP) {
-		_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_LINK_UP);
+		postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_UP);
 	}
 	if (netif->flags & NETIF_FLAG_LINK_UP) {
-		_push_callback_event(tap->_nwid, ZTS_EVENT_NETIF_LINK_DOWN);
+		postEvent(tap->_nwid, ZTS_EVENT_NETIF_LINK_DOWN);
 	}
 	//print_netif_info(netif);
 }
@@ -480,7 +485,7 @@ static err_t netif_init_6(struct netif *netif)
 	return ERR_OK;
 }

-void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier::InetAddress &ip)
+void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip)
 {
 	char ipbuf[INET6_ADDRSTRLEN];
 	char macbuf[ZTS_MAC_ADDRSTRLEN];
@@ -502,7 +507,7 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 			macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf));
 		netif_set_up(n);
 		netif_set_link_up(n);
-		ZeroTier::VirtualTap *vtap = (ZeroTier::VirtualTap*)tapref;
+		VirtualTap *vtap = (VirtualTap*)tapref;
 		vtap->netif4 = (void*)n;
 	}
 	if (ip.isV6())
@@ -525,7 +530,7 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
 		DEBUG_INFO("initialized netif as [mac=%s, addr=%s]", 
 			macbuf, ip.toString(ipbuf));
-		ZeroTier::VirtualTap *vtap = (ZeroTier::VirtualTap*)tapref;
+		VirtualTap *vtap = (VirtualTap*)tapref;
 		vtap->netif6 = (void*)n;
 	}
 	// Set netif callbacks, these will be used to inform decisions made
--- a/src/lwipDriver.hpp
+++ b/src/lwipDriver.hpp
@@ -0,0 +1,179 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * lwIP network stack driver
+ */
+
+#ifndef LIBZT_LWIP_DRIVER_HPP
+#define LIBZT_LWIP_DRIVER_HPP
+
+#include "Debug.hpp"
+#include "lwip/err.h"
+
+namespace ZeroTier {
+
+class MAC;
+class Mutex;
+class VirtualTap;
+struct InetAddress;
+
+/**
+ * @brief Structure used to associate packets with interfaces.
+ */
+struct zts_sorted_packet
+{
+	// lwIP pbuf containing packet (originally encapsulated by ZT packet)
+	struct pbuf *p;
+	// ZT VirtualTap from which this packet originates
+	VirtualTap *vtap;
+	// lwIP netif we should accept this packet on
+	struct netif *n;
+};
+
+/**
+ * @brief Return whether a given netif's NETIF_FLAG_UP flag is set
+ *
+ * @usage This is a convenience function to encapsulate a macro
+ */
+bool lwip_is_netif_up(void *netif);
+
+/**
+ * @brief Increase the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when we know the stack won't be used by any virtual taps
+ */
+void lwip_hibernate_driver();
+
+/**
+ * @brief Decrease the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when at least one virtual tap is active
+ */
+void lwip_wake_driver();
+
+/**
+ * @brief Initialize network stack semaphores, threads, and timers.
+ *
+ * @usage This is called during the initial setup of each VirtualTap but is only allowed to execute once
+ * @return
+ */
+void lwip_driver_init();
+
+/**
+ * @brief Shutdown the stack as completely as possible (not officially supported by lwIP)
+ *
+ * @usage This is to be called after it is determined that no further network activity will take place.
+ * The tcpip thread will be stopped, all interfaces will be brought down and all resources will 
+ * be deallocated. A full application restart will be required to bring the stack back online.
+ * @return
+ */
+void lwip_driver_shutdown();
+
+/**
+ * @brief Bring down and delete all interfaces belonging to the given virtual tap
+ *
+ * @usage This is to be called when the application desires to stop all traffic processing in the 
+ * stack. Unlike lwip_driver_shutdown(), the application can easily resume traffic processing 
+ * by re-adding a virtual tap (and associated lwip netifs)
+ * @return
+ */
+void lwip_dispose_of_netifs(void *tapref);
+
+/**
+ * @brief Initialize and start the DNS client
+ *
+ * @usage Called after lwip_driver_init()
+ * @return
+ */
+void lwip_dns_init();
+
+/**
+ * @brief Starts DHCP timers
+ *
+ * @usage lwip_driver_init()
+ * @return
+ */
+void lwip_start_dhcp(void *netif);
+
+/**
+ * @brief Called when the status of a netif changes:
+ *  - Interface is up/down (ZTS_EVENT_NETIF_UP, ZTS_EVENT_NETIF_DOWN)
+ *  - Address changes while up (ZTS_EVENT_NETIF_NEW_ADDRESS)
+ */
+static void netif_status_callback(struct netif *netif);
+
+/**
+ * @brief Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
+ */
+static void netif_remove_callback(struct netif *netif);
+
+/**
+ * @brief Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
+ */
+static void netif_link_callback(struct netif *netif);
+
+/**
+ * @brief Set up an interface in the network stack for the VirtualTap.
+ *
+ * @param
+ * @param tapref Reference to VirtualTap that will be responsible for sending and receiving data
+ * @param mac Virtual hardware address for this ZeroTier VirtualTap interface
+ * @param ip Virtual IP address for this ZeroTier VirtualTap interface
+ * @return
+ */
+void lwip_init_interface(void *tapref, const MAC &mac, const InetAddress &ip);
+
+/**
+ * @brief Called from the stack, outbound ethernet frames from the network stack enter the ZeroTier virtual wire here.
+ *
+ * @usage This shall only be called from the stack or the stack driver. Not the application thread.
+ * @param netif Transmits an outgoing Ethernet fram from the network stack onto the ZeroTier virtual wire
+ * @param p A pointer to the beginning of a chain pf struct pbufs
+ * @return
+ */
+err_t lwip_eth_tx(struct netif *netif, struct pbuf *p);
+
+/**
+ * @brief Receives incoming Ethernet frames from the ZeroTier virtual wire
+ *
+ * @usage This shall be called from the VirtualTap's I/O thread (via VirtualTap::put())
+ * @param tap Pointer to VirtualTap from which this data comes
+ * @param from Origin address (virtual ZeroTier hardware address)
+ * @param to Intended destination address (virtual ZeroTier hardware address)
+ * @param etherType Protocol type
+ * @param data Pointer to Ethernet frame
+ * @param len Length of Ethernet frame
+ * @return
+ */
+void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
+	const void *data, unsigned int len);
+
+} // namespace ZeroTier
+
+#endif // _H
--- a/src/lwipRawDriver.cpp
+++ b/src/lwipRawDriver.cpp
--- a/src/lwipRawDriver.hpp
+++ b/src/lwipRawDriver.hpp
--- a/src/lwipopts.h
+++ b/src/lwipopts.h