Improved reliability and performance, better startup and shutdown semantics, HTTP control plane is now disabled by default

CMakeLists.txt

@@ -1,6 +1,6 @@
 #
 # ZeroTier SDK - Network Virtualization Everywhere
-# Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
+# Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
 # 
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
 #  GNU General Public License for more details.
 # 
 #  You should have received a copy of the GNU General Public License
-#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
 # 
 #  --
 # 
@@ -100,17 +100,25 @@ if (BUILDING_WIN)
 	set(JAVA_INCLUDE_PATH "C:\\Program Files\\Java\\jdk-10.0.2\\include")
 endif ()
 
-find_package (JNI)
+set(JAVA_AWT_LIBRARY NotNeeded)
+set(JAVA_JVM_LIBRARY NotNeeded)
+set(JAVA_INCLUDE_PATH2 NotNeeded)
+set(JAVA_AWT_INCLUDE_PATH NotNeeded)
+find_package(JNI REQUIRED)
+
 if (JNI_FOUND)
-    message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+    message (STATUS "JNI_INCLUDE_DIR=${JNI_INCLUDE_DIRS}")
     message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
     list (GET JNI_INCLUDE_DIRS 0 JNI_INCLUDE_DIR)
     message (STATUS "jni path=${JNI_INCLUDE_DIR}")
     include_directories ("${JNI_INCLUDE_DIR}")
-    include_directories ("${JNI_INCLUDE_DIRS}")
+    #include_directories ("${JNI_INCLUDE_DIRS}")
 	if (BUILDING_WIN)
 		include_directories ("${JNI_INCLUDE_DIR}\\win32")
 	endif ()
+	if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") # on macOS
+		include_directories ("${JNI_INCLUDE_DIR}/darwin")
+	endif ()
 else ()
 	    message (STATUS "JNI not found")
 endif ()
@@ -156,7 +164,7 @@ endif ()
 # |                                    FLAGS                                  |
 # -----------------------------------------------------------------------------
 
-set (LIBZT_FLAGS "-DZT_SDK=1")
+set (LIBZT_FLAGS "-DZT_SDK=1 -D_USING_LWIP_DEFINITIONS_=1")
 set (LIBZT_FLAGS_DEBUG "-DZT_SDK=1 -DLIBZT_TRACE=1 -DLWIP_DEBUG=1 -DLIBZT_DEBUG=1 -DNS_TRACE=1 -DNS_DEBUG=1")
 
 set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${LIBZT_FLAGS_DEBUG}")
@@ -185,7 +193,8 @@ set (ZTO_SRC_DIR "${PROJ_DIR}/ext/ZeroTierOne")
 set (LIBZT_SRC_DIR "${PROJ_DIR}/src")
 
 include_directories ("${LIBZT_SRC_DIR}")
-include_directories ("${ZTO_SRC_DIR}/include")
+#include_directories ("${ZTO_SRC_DIR}/include")
+include_directories ("${PROJ_DIR}")
 include_directories ("${ZTO_SRC_DIR}/osdep")
 include_directories ("${ZTO_SRC_DIR}/node")
 include_directories ("${ZTO_SRC_DIR}/service")
@@ -223,7 +232,6 @@ file (GLOB ExampleAppSrcGlob
 # header globs for xcode frameworks
 file (GLOB frameworkPrivateHeaderGlob
 	${INCLUDE_PATH}/libzt.h
-	${INCLUDE_PATH}/libztDefs.h
 	${INCLUDE_PATH}/libztDebug.h)
 file (GLOB frameworkPublicHeaderGlob ${INCLUDE_PATH}/Xcode-Bridging-Header.h)
 file (GLOB frameworkHeaderGlob ${frameworkPublicHeaderGlob} ${frameworkPrivateHeaderGlob})
@@ -351,28 +359,28 @@ endif ()
 # -----------------------------------------------------------------------------
 
 if (SHOULD_BUILD_TESTS)
-	foreach (testsourcefile ${ExampleAppSrcGlob})
+#	foreach (testsourcefile ${ExampleAppSrcGlob})
-	    string (REPLACE ".cpp" "" testname ${testsourcefile})
+#	    string (REPLACE ".cpp" "" testname ${testsourcefile})
-	    get_filename_component (testname ${testname} NAME)
+#	    get_filename_component (testname ${testname} NAME)
-	    add_executable (${testname} ${testsourcefile})
+#	    add_executable (${testname} ${testsourcefile})
-	if (BUILDING_WIN)
+#	if (BUILDING_WIN)
-	    target_link_libraries (${testname} ${STATIC_LIB_NAME})
+#	    target_link_libraries (${testname} ${STATIC_LIB_NAME})
-	else ()
+#	else ()
-		target_link_libraries (${testname} ${STATIC_LIB_NAME} pthread dl)
+#		target_link_libraries (${testname} ${STATIC_LIB_NAME} pthread dl)
-	endif ()
+#	endif ()
-	endforeach (testsourcefile ${ExampleAppSrcGlob})
+#	endforeach (testsourcefile ${ExampleAppSrcGlob})
 
 if (NOT BUILDING_WIN) # only necessary for raw driver development
 	# selftest
-	add_executable (selftest ${PROJ_DIR}/test/selftest.cpp)
+	#add_executable (selftest ${PROJ_DIR}/test/selftest.cpp)
-	target_compile_options (selftest PRIVATE -D__SELFTEST__)
+	#target_compile_options (selftest PRIVATE -D__SELFTEST__)
-	if (BUILDING_WIN)
+	#if (BUILDING_WIN)
-		target_link_libraries (selftest ${STATIC_LIB_NAME} ${ws2_32_LIBRARY_PATH} ${shlwapi_LIBRARY_PATH} ${iphlpapi_LIBRARY_PATH})
+	#	target_link_libraries (selftest ${STATIC_LIB_NAME} ${ws2_32_LIBRARY_PATH} ${shlwapi_LIBRARY_PATH} ${iphlpapi_LIBRARY_PATH})
-	else ()
+	#else ()
-		target_link_libraries (selftest ${STATIC_LIB_NAME} pthread)
+	#	target_link_libraries (selftest ${STATIC_LIB_NAME} pthread)
-	endif ()
+	#endif ()
 	# nativetest
-	add_executable (nativetest ${PROJ_DIR}/test/selftest.cpp)
+	#add_executable (nativetest ${PROJ_DIR}/test/selftest.cpp)
-	target_compile_options (nativetest PRIVATE -D__NATIVETEST__)
+	#target_compile_options (nativetest PRIVATE -D__NATIVETEST__)
 endif ()
 endif ()

README.md

@@ -26,9 +26,7 @@ Java JNI API: [ZeroTier.java](packages/android/app/src/main/java/ZeroTier.java)
 ### C++ Example
 
 ```
-#include <sys/socket.h>
 #include <arpa/inet.h>
-#include <netinet/in.h>
 
 #include "libzt.h"
 
@@ -38,14 +36,15 @@ int main()
 	char *remoteIp = "10.8.8.42";
 	int remotePort = 8080;
 	int fd, err = 0;
-	struct sockaddr_in addr;
+	struct zts_sockaddr_in addr;
-	addr.sin_family = AF_INET;
+	addr.sin_family = ZTS_AF_INET;
 	addr.sin_addr.s_addr = inet_addr(remoteIp);
 	addr.sin_port = htons(remotePort);
 
 	zts_startjoin("path", 0xc7cd7c981b0f52a2); // config path, network ID
+	printf("nodeId=%llx\n", zts_get_node_id());
 
-	if ((fd = zts_socket(AF_INET, SOCK_STREAM, 0)) < 0) {
+	if ((fd = zts_socket(ZTS_AF_INET, ZTS_SOCK_STREAM, 0)) < 0) {
 		printf("error creating socket\n");
 	}
 	if ((err = zts_connect(fd, (const struct sockaddr *)&addr, sizeof(addr))) < 0) {

attic/SysUtils.cpp

@@ -30,15 +30,16 @@
  * Platform-specific implementations of common functions
  */
 
+/*
 #if defined(__linux__) || defined(__APPLE__)
 #include <sys/socket.h>
 #include <pthread.h>
 #endif
 
-#include "SysUtils.h"
 #include <stdint.h>
 
 #ifdef __linux__
 #include <sys/syscall.h>
 #include <unistd.h>
-#endif
+#endif
+*/

attic/Utilities.cpp

@@ -30,6 +30,7 @@
  * Misc utilities
  */
 
+/*
 #include "Utilities.h"
 
 #if defined(_WIN32_FALSE)
@@ -84,7 +85,9 @@ int inet_pton4(const char *src, void *dst)
 
 	return 1;
 }
+*/
 
+/*
 int inet_pton6(const char *src, void *dst)
 {
 	static const char xdigits[] = "0123456789abcdef";
@@ -94,7 +97,7 @@ int inet_pton6(const char *src, void *dst)
 	uint8_t *endp = tp + NS_IN6ADDRSZ;
 	uint8_t *colonp = NULL;
 
-	/* Leading :: requires some special handling. */
+	// Leading :: requires some special handling.
 	if (*src == ':')
 	{
 		if (*++src != ':')
@@ -144,7 +147,7 @@ int inet_pton6(const char *src, void *dst)
 		{
 			tp += NS_INADDRSZ;
 			saw_xdigit = 0;
-			break; /* '\0' was seen by inet_pton4(). */
+			break; // '\0' was seen by inet_pton4().
 		}
 		return 0;
 	}
@@ -157,10 +160,10 @@ int inet_pton6(const char *src, void *dst)
 	}
 	if (colonp != NULL)
 	{
-		/*
+		//
-		 * Since some memmove()'s erroneously fail to handle
+		// Since some memmove()'s erroneously fail to handle
-		 * overlapping regions, we'll do the shift by hand.
+		// overlapping regions, we'll do the shift by hand.
-		 */
+		//
 		const int n = tp - colonp;
 
 		if (tp == endp)
@@ -180,7 +183,9 @@ int inet_pton6(const char *src, void *dst)
 
 	return 1;
 }
+*/
 
+/*
 int inet_pton(int af, const char *src, void *dst)
 {
 	switch (af)
@@ -250,9 +255,12 @@ char *beautify_eth_proto_nums(int proto)
 	if (proto == 0x9100) return (char*)"VLAN-tagged (IEEE 802.1Q) frame with double tagging";
 	return (char*)"UNKNOWN";
 }
+*/
 
+/*
 void mac2str(char *macbuf, int len, unsigned char* addr)
 {
 	snprintf(macbuf, len, "%02x:%02x:%02x:%02x:%02x:%02x",
 		 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
 }
+*/

attic/ZT1Service.cpp

@@ -30,10 +30,9 @@
  * ZeroTier One service control wrapper
  */
 
+/*
 #include "libzt.h"
 #include "ZT1Service.h"
-#include "libztDebug.h"
-#include "SysUtils.h"
 
 #include "Phy.hpp"
 #include "OneService.hpp"
@@ -41,8 +40,16 @@
 #include "OSUtils.hpp"
 #include "Mutex.hpp"
 
+#include <pthread.h>
+*/
+
+
+/*
 std::vector<void*> vtaps;
 ZeroTier::Mutex _vtaps_lock;
+ZeroTier::Mutex _service_lock;
+*/
+/*
 
 #ifdef __cplusplus
 extern "C" {
@@ -56,6 +63,8 @@ std::string netDir;  // Where network .conf files are to be written
 ZeroTier::Mutex _multiplexer_lock;
 
 int servicePort = LIBZT_DEFAULT_PORT;
+bool _freeHasBeenCalled = false;
+bool _serviceIsShuttingDown = false;
 
 #if defined(_WIN32)
 WSADATA wsaData;
@@ -64,9 +73,11 @@ WSADATA wsaData;
 
 void api_sleep(int interval_ms);
 
-/****************************************************************************/
+pthread_t service_thread;
-/* ZeroTier Core helper functions for libzt - DON'T CALL THESE DIRECTLY     */
+
-/****************************************************************************/
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Core helper functions for libzt - DON'T CALL THESE DIRECTLY     //
+//////////////////////////////////////////////////////////////////////////////
 
 std::vector<ZT_VirtualNetworkRoute> *zts_get_network_routes(const uint64_t nwid)
 {
@@ -95,15 +106,15 @@ VirtualTap *getTapByAddr(ZeroTier::InetAddress *addr)
 		// check address schemes
 		for (int j=0; j<(int)(s->_ips.size()); j++) {
 			if ((s->_ips[j].isV4() && addr->isV4()) || (s->_ips[j].isV6() && addr->isV6())) {
-				/* DEBUG_EXTRA("looking at tap %s, <addr=%s> --- for <%s>", s->_dev.c_str(),
+				// DEBUG_INFO("looking at tap %s, <addr=%s> --- for <%s>", s->_dev.c_str(),
-				s->_ips[j].toString(ipbuf), addr->toIpString(ipbuf2)); */
+				// s->_ips[j].toString(ipbuf), addr->toIpString(ipbuf2));
 				if (s->_ips[j].isEqualPrefix(addr)
 					|| s->_ips[j].ipsEqual(addr)
 					|| s->_ips[j].containsAddress(addr)
 					|| (addr->isV6() && _ipv6_in_subnet(&s->_ips[j], addr))
 					)
 				{
-					//DEBUG_EXTRA("selected tap %s, <addr=%s>", s->_dev.c_str(), s->_ips[j].toString(ipbuf));
+					//DEBUG_INFO("selected tap %s, <addr=%s>", s->_dev.c_str(), s->_ips[j].toString(ipbuf));
 					_vtaps_lock.unlock();
 					return s;
 				}
@@ -118,8 +129,8 @@ VirtualTap *getTapByAddr(ZeroTier::InetAddress *addr)
 				nm = target.netmask();
 				via = managed_routes->at(i).via;
 				if (target.containsAddress(addr)) {
-					/* DEBUG_EXTRA("chose tap with route <target=%s, nm=%s, via=%s>", target.toString(ipbuf),
+					// DEBUG_INFO("chose tap with route <target=%s, nm=%s, via=%s>", target.toString(ipbuf),
-					nm.toString(ipbuf2), via.toString(ipbuf3)); */
+					// nm.toString(ipbuf2), via.toString(ipbuf3));
 					_vtaps_lock.unlock();
 					return s;
 				}
@@ -169,31 +180,27 @@ VirtualTap *getAnyTap()
 	return vtap;
 }
 
-uint64_t zts_get_node_id_from_file(const char *filepath)
-{
-	std::string fname("identity.public");
-	std::string fpath(filepath);
-	std::string oldid;
-	if (ZeroTier::OSUtils::fileExists((fpath + ZT_PATH_SEPARATOR_S + fname).c_str(), false)) {
-		ZeroTier::OSUtils::readFile((fpath + ZT_PATH_SEPARATOR_S + fname).c_str(), oldid);
-		return Utils::hexStrToU64(oldid.c_str());
-	}
-	return 0;
-}
-
 // Starts a ZeroTier service in the background
 #if defined(_WIN32)
 DWORD WINAPI zts_start_service(LPVOID thread_id)
 #else
 void *zts_start_service(void *thread_id)
 #endif
-{
+{	
-	DEBUG_INFO("zto-thread, path=%s", homeDir.c_str());
+	void *retval;
-	// Where network .conf files will be stored
+	DEBUG_INFO("identities are stored in path (%s)", homeDir.c_str());
 	netDir = homeDir + "/networks.d";
 	zt1Service = (ZeroTier::OneService *)0;
-	// Construct path for network config and supporting service files
+
-	if (homeDir.length()) {
+	if (!homeDir.length()) {
+		DEBUG_ERROR("homeDir is empty, could not construct path");
+		retval = NULL;
+	} if (zt1Service) {
+		DEBUG_INFO("service already started, doing nothing");
+		retval = NULL;
+	}
+
+	try {
 		std::vector<std::string> hpsp(ZeroTier::OSUtils::split(homeDir.c_str(), ZT_PATH_SEPARATOR_S,"",""));
 		std::string ptmp;
 		if (homeDir[0] == ZT_PATH_SEPARATOR) {
@@ -211,48 +218,41 @@ void *zts_start_service(void *thread_id)
 				}
 			}
 		}
-	}
+		for(;;) {
-	else {
+			_service_lock.lock();
-		DEBUG_ERROR("homeDir is empty, could not construct path");
+			zt1Service = OneService::newInstance(homeDir.c_str(),servicePort);
-		return NULL;
+			_service_lock.unlock();
-	}
+			switch(zt1Service->run()) {
-	if (servicePort <= 0) {
+				case OneService::ONE_STILL_RUNNING: // shouldn't happen, run() won't return until done
-		DEBUG_INFO("no port specified, will bind to random port. use zts_set_service_port() if you want.");
+				case OneService::ONE_NORMAL_TERMINATION:
-	}
+					break;
-	else {
+				case OneService::ONE_UNRECOVERABLE_ERROR:
-		DEBUG_INFO("binding to port=%d", servicePort);
+					fprintf(stderr,"fatal error: %s" ZT_EOL_S,zt1Service->fatalErrorMessage().c_str());
-	}
+					break;
-	for (;;) {
+				case OneService::ONE_IDENTITY_COLLISION: {
-		zt1Service = ZeroTier::OneService::newInstance(homeDir.c_str(),servicePort);
+					delete zt1Service;
-		switch(zt1Service->run()) {
+					zt1Service = (OneService *)0;
-			case ZeroTier::OneService::ONE_STILL_RUNNING:
+					std::string oldid;
-			case ZeroTier::OneService::ONE_NORMAL_TERMINATION:
+					OSUtils::readFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid);
-				break;
+					if (oldid.length()) {
-			case ZeroTier::OneService::ONE_UNRECOVERABLE_ERROR:
+						OSUtils::writeFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid);
-				DEBUG_ERROR("fatal error: %s",zt1Service->fatalErrorMessage().c_str());
+						OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
-				break;
+						OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
-			case ZeroTier::OneService::ONE_IDENTITY_COLLISION: {
+					}
-				delete zt1Service;
+				}	continue; // restart!
-				zt1Service = (ZeroTier::OneService *)0;
-				std::string oldid;
-				ZeroTier::OSUtils::readFile((homeDir + ZT_PATH_SEPARATOR_S
-					+ "identity.secret").c_str(),oldid);
-				if (oldid.length()) {
-					ZeroTier::OSUtils::writeFile((homeDir + ZT_PATH_SEPARATOR_S
-						+ "identity.secret.saved_after_collision").c_str(),oldid);
-					ZeroTier::OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S
-						+ "identity.secret").c_str());
-					ZeroTier::OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S
-						+ "identity.public").c_str());
-				}
 			}
-			continue; // restart!
+			break; // terminate loop -- normally we don't keep restarting
 		}
-		break; // terminate loop -- normally we don't keep restarting
+		_serviceIsShuttingDown = true;
+		_service_lock.lock();
+		delete zt1Service;
+		zt1Service = (OneService *)0;
+		_service_lock.unlock();
+		_serviceIsShuttingDown = false;
+	} catch ( ... ) {
+		fprintf(stderr,"unexpected exception starting main OneService instance" ZT_EOL_S);
 	}
-	delete zt1Service;
+	pthread_exit(NULL);
-	zt1Service = (ZeroTier::OneService *)0;
-	return NULL;
 }
 
 int zts_get_num_assigned_addresses(const uint64_t nwid)
@@ -264,7 +264,7 @@ int zts_get_num_assigned_addresses(const uint64_t nwid)
 	if (!tap) {
 		return -1;
 	}
-	int sz = -1;
+	int sz;
 	_vtaps_lock.lock();
 	sz = tap->_ips.size();
 	_vtaps_lock.unlock();
@@ -292,18 +292,32 @@ int zts_get_address_at_index(
 	return err;
 }
 
-/****************************************************************************/
+//////////////////////////////////////////////////////////////////////////////
-/* ZeroTier Service Controls                                                */
+// ZeroTier Service Controls                                                //
-/****************************************************************************/
+//////////////////////////////////////////////////////////////////////////////
 
-int zts_set_service_port(int portno)
+zts_err_t zts_set_service_port(int portno)
 {
-	if (portno > -1 && portno < 65535) {
+	zts_err_t retval = ZTS_ERR_OK;
-		// 0 is allowed, signals zt service to bind to a random port
+	_service_lock.lock();
-		servicePort = portno;
+	if (zt1Service) {
-		return 0;
+		DEBUG_INFO("please stop service before attempting to change port");
+		retval = ZTS_ERR_SERVICE;
 	}
-	return -1;
+	else {
+		if (portno > -1 && portno < ZTS_MAX_PORT) {
+			// 0 is allowed, signals to ZT service to bind to a random port
+			servicePort = portno;
+			retval = ZTS_ERR_OK;
+		}
+	}
+	_service_lock.unlock();
+	return retval;
+}
+
+int zts_get_service_port()
+{
+	return servicePort;
 }
 
 int zts_get_address(const uint64_t nwid, struct sockaddr_storage *addr, 
@@ -311,7 +325,7 @@ int zts_get_address(const uint64_t nwid, struct sockaddr_storage *addr,
 {
 	int err = -1;
 	if (!zt1Service) {
-		return -1;
+		return ZTS_ERR_SERVICE;
 	}
 	VirtualTap *tap = getTapByNWID(nwid);
 	if (!tap) {
@@ -368,49 +382,99 @@ void zts_get_rfc4193_addr(struct sockaddr_storage *addr, const uint64_t nwid, co
 	memcpy(addr, _rfc4193Addr.rawIpData(), sizeof(struct sockaddr_storage));
 }
 
-int zts_join(const uint64_t nwid)
+zts_err_t zts_join(const uint64_t nwid, int blocking)
 {
-	DEBUG_INFO("joining %llx", (unsigned long long)nwid);
+	zts_err_t retval = ZTS_ERR_OK;
-	if (nwid == 0) {
+	_service_lock.lock();
-		return -1;
+	if (blocking) {
+		if (!zt1Service) {
+			retval = ZTS_ERR_SERVICE;
+		} else {
+			while (!_zts_node_online()) {
+				if (_serviceIsShuttingDown) {
+					retval = ZTS_ERR_SERVICE;
+					break;
+				}
+				api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
+			}
+		}
+	} else {
+		if (!zt1Service || !_zts_node_online()) {
+			retval = ZTS_ERR_SERVICE;
+		}
 	}
-	if (zt1Service) {
+	if (!retval) {
-		zt1Service->join(nwid);
+		DEBUG_INFO("joining %llx", (unsigned long long)nwid);
+		if (nwid == 0) {
+			retval = ZTS_ERR_INVALID_ARG;
+		}
+		if (zt1Service) {
+			zt1Service->join(nwid);
+		}
+		// provide ZTO service reference to virtual taps
+		// TODO: This might prove to be unreliable, but it works for now
+		_vtaps_lock.lock();
+		for (size_t i=0;i<vtaps.size(); i++) {
+			VirtualTap *s = (VirtualTap*)vtaps[i];
+			s->zt1ServiceRef=(void*)zt1Service;
+		}
+		_vtaps_lock.unlock();
 	}
-	// provide ZTO service reference to virtual taps
+	_service_lock.unlock();
-	// TODO: This might prove to be unreliable, but it works for now
+	return retval;
-	_vtaps_lock.lock();
-	for (size_t i=0;i<vtaps.size(); i++) {
-		VirtualTap *s = (VirtualTap*)vtaps[i];
-		s->zt1ServiceRef=(void*)zt1Service;
-	}
-	_vtaps_lock.unlock();
-	return 0;
 }
 
-int zts_leave(const uint64_t nwid)
+zts_err_t zts_leave(const uint64_t nwid, int blocking)
 {
-	DEBUG_INFO("leaving %llx", (unsigned long long)nwid);
+	zts_err_t retval = ZTS_ERR_OK;
-	if (nwid == 0) {
+	_service_lock.lock();
-		return -1;
+	if (blocking) {
+		if (!zt1Service) {
+			retval = ZTS_ERR_SERVICE;
+		} else {
+			while (!_zts_node_online()) {
+				if (_serviceIsShuttingDown) {
+					retval = ZTS_ERR_SERVICE;
+					break;
+				}
+				api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
+			}
+		}
+	} else {
+		if (!zt1Service || !_zts_node_online()) {
+			retval = ZTS_ERR_SERVICE;
+		}
 	}
-	if (zt1Service) {
+	if (!retval) {
-		zt1Service->leave(nwid);
+		DEBUG_INFO("leaving %llx", (unsigned long long)nwid);
+		if (nwid == 0) {
+			retval = ZTS_ERR_INVALID_ARG;
+		}
+		if (zt1Service) {
+			zt1Service->leave(nwid);
+		}
 	}
-	return 0;
+	_service_lock.unlock();
+	return retval;
 }
 
 int zts_core_running()
 {
-	return zt1Service == NULL ? false : zt1Service->isRunning();
+	_service_lock.lock();
+	int retval = zt1Service == NULL ? false : zt1Service->isRunning();
+	_service_lock.unlock();
+	return retval;
 }
 
 int zts_stack_running()
 {
+	// PENDING: what if no networks are joined, the stack is still running. semantics need to change here
+	_service_lock.lock();
 	_vtaps_lock.lock();
-	// TODO: Perhaps a more robust way to check for this
+	// PENDING: Perhaps a more robust way to check for this
 	int running = vtaps.size() > 0 ? true : false;
 	_vtaps_lock.unlock();
+	_service_lock.unlock();
 	return running;
 }
 
@@ -419,47 +483,64 @@ int zts_ready()
 	return zts_core_running() && zts_stack_running();
 }
 
-int zts_start(const char *path, int blocking = false)
+zts_err_t zts_start(const char *path, int blocking = false)
 {
+	zts_err_t retval = ZTS_ERR_OK;
 	if (zt1Service) {
-		return 0; // already initialized, ok
+		return ZTS_ERR_SERVICE; // already initialized
+	}
+	if (_freeHasBeenCalled) {
+		return ZTS_ERR_INVALID_OP; // stack (presumably lwIP) has been dismantled, an application restart is required now
 	}
 	if (path) {
 		homeDir = path;
 	}
-	int err = 0;
 #if defined(_WIN32)
 		WSAStartup(MAKEWORD(2, 2), &wsaData); // initialize WinSock. Used in Phy for loopback pipe
 		HANDLE thr = CreateThread(NULL, 0, zts_start_service, NULL, 0, NULL);
 #else
-		pthread_t service_thread;
+		retval = pthread_create(&service_thread, NULL, zts_start_service, NULL);
-		err = pthread_create(&service_thread, NULL, zts_start_service, NULL);
+		// PENDING: Wait for confirmation that the ZT service has been initialized, 
+		// this wait condition is so brief and so rarely used that it should be
+		// acceptable even in a non-blocking context.
+		while(!zt1Service) { 
+			api_sleep(10);
+		}
 #endif
 
 	if (blocking) { // block to prevent service calls before we're ready
 		ZT_NodeStatus status;
 		status.online = 0;
-		DEBUG_EXTRA("waiting for zerotier service thread to start");
+		DEBUG_INFO("waiting for zerotier service thread to start");
 		while (zts_core_running() == false || zt1Service->getNode() == NULL) {
 			api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
 		}
-		DEBUG_EXTRA("waiting for node address assignment");
+		DEBUG_INFO("waiting for node address assignment");
 		while (zt1Service->getNode()->address() <= 0) {
 			api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
 		}
-		DEBUG_EXTRA("node=%llx", (unsigned long long)zts_get_node_id());
+		DEBUG_INFO("waiting for node to come online. ensure the node is authorized to join the network");
-		DEBUG_EXTRA("waiting for node to come online. ensure the node is authorized to join the network");
+		while (true) {
-		while (status.online <= 0) {
+			_service_lock.lock();
+			if (zt1Service && zt1Service->getNode() && zt1Service->getNode()->online()) {
+				DEBUG_INFO("node is fully online");
+				_service_lock.unlock();
+				break;
+			}
 			api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
-			zt1Service->getNode()->status(&status);
+			_service_lock.unlock();
 		}
+		DEBUG_INFO("node=%llx", (unsigned long long)zts_get_node_id());
 	}
-	return err;
+	return retval;
 }
 
-int zts_startjoin(const char *path, const uint64_t nwid)
+zts_err_t zts_startjoin(const char *path, const uint64_t nwid)
 {
-	int err = zts_start(path, true);
+	zts_err_t retval = ZTS_ERR_OK;
+	if ((retval = zts_start(path, true)) < 0) {
+		return retval;
+	}
 	while (true) {
 		try {
 			zts_join(nwid);
@@ -474,45 +555,111 @@ int zts_startjoin(const char *path, const uint64_t nwid)
 	while (zts_has_address(nwid) == false) {
 		api_sleep(ZTO_WRAPPER_CHECK_INTERVAL);
 	}
-	return err;
+	return retval;
 }
 
-void zts_stop()
+zts_err_t zts_stop(int blocking)
 {
+	zts_err_t ret = ZTS_ERR_OK;
+	_service_lock.lock();
+	VirtualTap *s;
 	if (zt1Service) {
 		zt1Service->terminate();
-		// disableTaps();
+		vtaps.clear();
+	}
+	else {
+		ret = ZTS_ERR_SERVICE; // nothing to do
 	}
 #if defined(_WIN32)
 	WSACleanup();
 #endif
+	_service_lock.unlock();
+	if (blocking) {
+		// block until service thread successfully exits
+		pthread_join(service_thread, NULL);
+	}
+	return ret;
 }
 
-void zts_get_path(char *homePath, size_t len)
+zts_err_t zts_free()
 {
-	if (homeDir.length()) {
+	zts_err_t retval = 0;
-		memset(homePath, 0, len);
+	_service_lock.lock();
-		size_t buf_len = len < homeDir.length() ? len : homeDir.length();
+	if (_freeHasBeenCalled) {
-		memcpy(homePath, homeDir.c_str(), buf_len);
+		retval = ZTS_ERR_INVALID_OP;
+		_service_lock.unlock();
+	} else {
+		_freeHasBeenCalled = true;
+		_service_lock.unlock();
+		retval = zts_stop();
 	}
+	// PENDING: add stack shutdown logic
+	return retval;
+}
+
+zts_err_t zts_get_path(char *homePath, size_t *len)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!homePath || *len <= 0 || *len > ZT_HOME_PATH_MAX_LEN) {
+		*len = 0; // signal that nothing was copied to the buffer
+		retval = ZTS_ERR_INVALID_ARG;
+	} else if (homeDir.length()) {
+		memset(homePath, 0, *len);
+		size_t buf_len = *len < homeDir.length() ? *len : homeDir.length();
+		memcpy(homePath, homeDir.c_str(), buf_len);
+		*len = buf_len;
+	}
+	return retval;
 }
 
 uint64_t zts_get_node_id()
 {
-	if (zt1Service) {
+	uint64_t nodeId = 0;
-		return zt1Service->getNode()->address();
+	_service_lock.lock();
+	if (_can_perform_service_operation()) {
+		nodeId = zt1Service->getNode()->address();
+	}
+	_service_lock.unlock();
+	return nodeId;
+}
+
+uint64_t zts_get_node_id_from_file(const char *filepath)
+{
+	std::string fname("identity.public");
+	std::string fpath(filepath);
+	std::string oldid;
+	if (ZeroTier::OSUtils::fileExists((fpath + ZT_PATH_SEPARATOR_S + fname).c_str(), false)) {
+		ZeroTier::OSUtils::readFile((fpath + ZT_PATH_SEPARATOR_S + fname).c_str(), oldid);
+		return Utils::hexStrToU64(oldid.c_str());
 	}
 	return 0;
 }
 
-unsigned long zts_get_peer_count()
+int zts_get_peer_count()
 {
-	if (zt1Service) {
+	unsigned int peerCount = 0;
-		return zt1Service->getNode()->peers()->peerCount;
+	_service_lock.lock();
-	}
+	if (_can_perform_service_operation()) {
-	else {
+		peerCount = zt1Service->getNode()->peers()->peerCount;
-		return 0;
+	} else {
+		peerCount = ZTS_ERR_SERVICE;
 	}
+	_service_lock.unlock();
+	return peerCount;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// Internal ZeroTier Service Controls (user application shall not use these)//
+//////////////////////////////////////////////////////////////////////////////
+
+int _zts_node_online()
+{
+	return zt1Service && zt1Service->getNode() && zt1Service->getNode()->online();
+}
+
+int _can_perform_service_operation()
+{
+	return zt1Service && zt1Service->isRunning() && zt1Service->getNode() && zt1Service->getNode()->online() && !_serviceIsShuttingDown;
 }
 
 bool _ipv6_in_subnet(ZeroTier::InetAddress *subnet, ZeroTier::InetAddress *addr)
@@ -561,3 +708,5 @@ void api_sleep(int interval_ms)
 #ifdef __cplusplus
 }
 #endif
+
+*/

attic/ZT1Service.h

@@ -30,64 +30,4 @@
  * ZeroTier One service control wrapper header file
  */
 
-#if defined(_WIN32)
-#include <Windows.h>
-#endif
 
-#include "ZeroTierOne.h"
-#include "InetAddress.hpp"
-#include "libztDefs.h"
-
-#include <vector>
-
-#ifndef ZT1SERVICE_H
-#define ZT1SERVICE_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-class VirtualTap;
-class VirtualSocket;
-
-VirtualTap *getTapByAddr(ZeroTier::InetAddress *addr);
-VirtualTap *getTapByName(char *ifname);
-VirtualTap *getTapByIndex(size_t index);
-VirtualTap *getAnyTap();
-
-/**
- * @brief Returns a vector of network routes { target, via, metric, etc... }
- *
- * @usage
- * @param nwid 16-digit hexidecimal network identifier
- * @return
- */
-std::vector<ZT_VirtualNetworkRoute> *zts_get_network_routes(const uint64_t nwid);
-
-/**
- * @brief Starts a ZeroTier service in the background
- *
- * @usage For internal use only.
- * @param
- * @return
- */
-#if defined(_WIN32)
-DWORD WINAPI zts_start_service(LPVOID thread_id);
-#else
-void *zts_start_service(void *thread_id);
-#endif
-
-/**
- * @brief Returns masked address for subnet comparisons
- *
- * @usage For internal use only.
- * @param socket_type
- * @return
- */
-bool _ipv6_in_subnet(ZeroTier::InetAddress *subnet, ZeroTier::InetAddress *addr);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // _H

attic/libztJNI.cpp

@@ -42,7 +42,6 @@
 #endif
 
 #include "libzt.h"
-#include "libztDefs.h"
 
 #include <jni.h>
 
@@ -57,10 +56,22 @@ namespace ZeroTier {
 	void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set);
 	void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set);
 
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
-	/* ZeroTier service controls                                                */
+	// ZeroTier service controls                                                //
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
+/*
+	JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_set_1service_1port(
+		JNIEnv *env, jobject thisObj, jint port)
+	{
+		zts_set_service_port(port);
+	}
 
+	JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_1service_1port(
+		JNIEnv *env, jobject thisObj, jint port)
+	{
+		return zts_get_service_port();
+	}
+*/
 	JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_start(
 		JNIEnv *env, jobject thisObj, jstring path, jboolean blocking)
 	{
@@ -87,6 +98,12 @@ namespace ZeroTier {
 		zts_stop();
 	}
 
+	JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_free(
+		JNIEnv *env, jobject thisObj)
+	{
+		zts_free();
+	}
+
 	JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_core_1running(
 		JNIEnv *env, jobject thisObj)
 	{
@@ -105,6 +122,12 @@ namespace ZeroTier {
 		return zts_ready();
 	}
 
+	JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_1service_1port(
+		JNIEnv *env, jobject thisObj, jint port)
+	{
+		return zts_get_num_joined_networks();
+	}xxx
+
 	JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_join(
 		JNIEnv *env, jobject thisObj, jlong nwid)
 	{
@@ -184,9 +207,9 @@ namespace ZeroTier {
 		return zts_get_peer_count();
 	}
 
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
-	/* ZeroTier Socket API                                                      */
+	// ZeroTier Socket API                                                      //
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
 
 	JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
 		JNIEnv *env, jobject thisObj, jint family, jint type, jint protocol)
@@ -207,7 +230,6 @@ namespace ZeroTier {
 		JNIEnv *env, jobject thisObj, jint fd, jobject addr)
 	{
 		struct sockaddr_storage ss;
-		int err;
 		zta2ss(env, &ss, addr);
 		socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
 		return zts_bind(fd, (struct sockaddr*)&ss, addrlen);
@@ -430,9 +452,9 @@ void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_
 	return;
 }
 
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
-	/* Helpers (for moving data across the JNI barrier)                         */
+	// Helpers (for moving data across the JNI barrier)                         //
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
 
 void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 {

include/Constants.hpp

@@ -0,0 +1,288 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Useful constants
+ */
+
+#ifndef LIBZT_CONSTANTS_HPP
+#define LIBZT_CONSTANTS_HPP
+
+//////////////////////////////////////////////////////////////////////////////
+// Error codes returned by libzt API                                        //
+//////////////////////////////////////////////////////////////////////////////
+
+typedef int zts_err_t;
+
+#define ZTS_ERR_OK 0
+#define ZTS_ERR_INVALID_ARG -1 // A parameter provided by the user application is invalid (e.g. our of range, NULL, etc)
+#define ZTS_ERR_SERVICE -2 // The service isn't initialized or is for some other reason currently unavailable
+#define ZTS_ERR_INVALID_OP -3 // For some reason this API operation is not permitted (perhaps the service is still starting?)
+
+//////////////////////////////////////////////////////////////////////////////
+// libzt config                                                             //
+//////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Default port that libzt will use to support all virtual communication
+ */
+#define ZTS_DEFAULT_PORT 9994
+
+/**
+ * Maximum port number allowed
+ */
+#define ZTS_MAX_PORT 65535
+
+/**
+ * For layer-2 only (this will omit all user-space network stack code)
+ */
+#define ZTS_NO_STACK 0
+
+/**
+ * How fast service states are re-checked (in milliseconds)
+ */
+#define ZTS_WRAPPER_CHECK_INTERVAL 50
+
+/**
+ * By how much thread I/O and callback loop delays are multiplied (unitless)
+ */
+#define ZTS_HIBERNATION_MULTIPLIER 50
+
+/**
+ * Maximum allowed number of networks joined to concurrently
+ */
+#define ZTS_MAX_JOINED_NETWORKS 64
+
+/**
+ * Maximum address assignments per network
+ */
+#define ZTS_MAX_ASSIGNED_ADDRESSES 16
+
+/**
+ * Maximum routes per network
+ */
+#define ZTS_MAX_NETWORK_ROUTES 32
+
+/**
+ * Length of buffer required to hold a ztAddress/nodeID
+ */
+#define ZTS_ID_LEN 16
+
+/**
+ * Polling interval (in ms) for file descriptors wrapped in the Phy I/O loop (for raw drivers only)
+ */
+#define ZTS_PHY_POLL_INTERVAL 1
+
+/**
+ * Maximum length of libzt/ZeroTier home path (where keys, and config files are stored)
+ */
+#define ZTS_HOME_PATH_MAX_LEN 256
+
+/**
+ * Length of human-readable MAC address string
+ */
+#define ZTS_MAC_ADDRSTRLEN 18
+
+/**
+ * Interval (in ms) for performing background tasks
+ */
+#define ZTS_HOUSEKEEPING_INTERVAL 1000
+
+//////////////////////////////////////////////////////////////////////////////
+// lwIP driver config                                                       //
+// For more LWIP configuration options see: include/lwipopts.h              //
+//////////////////////////////////////////////////////////////////////////////
+
+/* 
+ * The following three quantities are related and govern how incoming frames are fed into the 
+ * network stack's core:
+
+ * Every LWIP_GUARDED_BUF_CHECK_INTERVAL milliseconds, a callback will be called from the core and 
+ * will input a maximum of LWIP_FRAMES_HANDLED_PER_CORE_CALL frames before returning control back
+ * to the core. Meanwhile, incoming frames from the ZeroTier wire will be allocated and their 
+ * pointers will be cached in the receive frame buffer of the size LWIP_MAX_GUARDED_RX_BUF_SZ to 
+ * await the next callback from the core
+ */
+
+#define LWIP_GUARDED_BUF_CHECK_INTERVAL 5 // in ms
+#define LWIP_MAX_GUARDED_RX_BUF_SZ 1024 // number of frame pointers that can be cached waiting for receipt into core
+#define LWIP_FRAMES_HANDLED_PER_CORE_CALL 16 // How many frames are handled per call from core
+
+typedef signed char err_t;
+
+#define ND6_DISCOVERY_INTERVAL 1000
+#define ARP_DISCOVERY_INTERVAL ARP_TMR_INTERVAL
+
+//////////////////////////////////////////////////////////////////////////////
+// Subset of: ZeroTierOne.h and Constants.hpp                               //
+// We redefine a few ZT structures here so that we don't need to drag the   //
+// entire ZeroTierOne.h file into the user application                      //
+//////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Maximum MTU size for ZeroTier
+ */
+#define ZT_MAX_MTU 10000
+
+/**
+ * Maximum number of direct network paths to a given peer
+ */
+#define ZT_MAX_PEER_NETWORK_PATHS 16
+
+//
+// This include file also auto-detects and canonicalizes some environment
+// information defines:
+//
+// __LINUX__
+// __APPLE__
+// __BSD__ (OSX also defines this)
+// __UNIX_LIKE__ (Linux, BSD, etc.)
+// __WINDOWS__
+//
+// Also makes sure __BYTE_ORDER is defined reasonably.
+//
+
+// Hack: make sure __GCC__ is defined on old GCC compilers
+#ifndef __GCC__
+#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) || defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) || defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
+#define __GCC__
+#endif
+#endif
+
+#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+#ifndef __LINUX__
+#define __LINUX__
+#endif
+#ifndef __UNIX_LIKE__
+#define __UNIX_LIKE__
+#endif
+#include <endian.h>
+#endif
+
+#ifdef __APPLE__
+#define likely(x) __builtin_expect((x),1)
+#define unlikely(x) __builtin_expect((x),0)
+#include <TargetConditionals.h>
+#ifndef __UNIX_LIKE__
+#define __UNIX_LIKE__
+#endif
+#ifndef __BSD__
+#define __BSD__
+#endif
+#include <machine/endian.h>
+#endif
+
+// Defined this macro to disable "type punning" on a number of targets that
+// have issues with unaligned memory access.
+#if defined(__arm__) || defined(__ARMEL__) || (defined(__APPLE__) && ( (defined(TARGET_OS_IPHONE) && (TARGET_OS_IPHONE != 0)) || (defined(TARGET_OS_WATCH) && (TARGET_OS_WATCH != 0)) || (defined(TARGET_IPHONE_SIMULATOR) && (TARGET_IPHONE_SIMULATOR != 0)) ) )
+#ifndef ZT_NO_TYPE_PUNNING
+#define ZT_NO_TYPE_PUNNING
+#endif
+#endif
+
+#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__)
+#ifndef __UNIX_LIKE__
+#define __UNIX_LIKE__
+#endif
+#ifndef __BSD__
+#define __BSD__
+#endif
+#include <machine/endian.h>
+#ifndef __BYTE_ORDER
+#define __BYTE_ORDER _BYTE_ORDER
+#define __LITTLE_ENDIAN _LITTLE_ENDIAN
+#define __BIG_ENDIAN _BIG_ENDIAN
+#endif
+#endif
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef __WINDOWS__
+#define __WINDOWS__
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#pragma warning(disable : 4290)
+#pragma warning(disable : 4996)
+#pragma warning(disable : 4101)
+#undef __UNIX_LIKE__
+#undef __BSD__
+#define ZT_PATH_SEPARATOR '\\'
+#define ZT_PATH_SEPARATOR_S "\\"
+#define ZT_EOL_S "\r\n"
+#include <WinSock2.h>
+#include <Windows.h>
+#endif
+
+// Assume little endian if not defined
+#if (defined(__APPLE__) || defined(__WINDOWS__)) && (!defined(__BYTE_ORDER))
+#undef __BYTE_ORDER
+#undef __LITTLE_ENDIAN
+#undef __BIG_ENDIAN
+#define __BIG_ENDIAN 4321
+#define __LITTLE_ENDIAN 1234
+#define __BYTE_ORDER 1234
+#endif
+
+#ifdef __UNIX_LIKE__
+#define ZT_PATH_SEPARATOR '/'
+#define ZT_PATH_SEPARATOR_S "/"
+#define ZT_EOL_S "\n"
+#endif
+
+#ifndef __BYTE_ORDER
+#include <endian.h>
+#endif
+
+#ifdef __NetBSD__
+#define RTF_MULTICAST   0x20000000
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
+#ifndef likely
+#define likely(x) __builtin_expect((x),1)
+#endif
+#ifndef unlikely
+#define unlikely(x) __builtin_expect((x),0)
+#endif
+#else
+#ifndef likely
+#define likely(x) (x)
+#endif
+#ifndef unlikely
+#define unlikely(x) (x)
+#endif
+#endif
+
+#ifdef __WINDOWS__
+#define ZT_PACKED_STRUCT(D) __pragma(pack(push,1)) D __pragma(pack(pop))
+#else
+#define ZT_PACKED_STRUCT(D) D __attribute__((packed))
+#endif
+
+#endif // _H

include/Debug.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -33,19 +33,21 @@
 #ifndef LIBZT_DEBUG_HPP
 #define LIBZT_DEBUG_HPP
 
+//////////////////////////////////////////////////////////////////////////////
+// Debugging Macros                                                         //
+//////////////////////////////////////////////////////////////////////////////
+
 #if defined(__linux__) || defined(__APPLE__)
 #include <sys/syscall.h>
 #include <pthread.h>
 #include <unistd.h>
 #endif
-
 #include <string.h>
 
 #define ZT_MSG_ERROR       true // Errors
 #define ZT_MSG_INFO        true // Information which is generally useful to any developer
 #define ZT_MSG_TEST        true // For use in selftest
 #define ZT_MSG_TRANSFER    true // RX/TX specific statements
-#define ZT_MSG_EXTRA       true // If nothing in your world makes sense
 
 #define ZT_COLOR           true
 
@@ -75,16 +77,6 @@
 
 #define ZT_FILENAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) // short
 
-#ifdef __linux__
-  #define ZT_THREAD_ID syscall(SYS_gettid)
-#endif
-#ifdef __APPLE__
-  #define ZT_THREAD_ID (long)0
-#endif
-#ifdef _WIN32
-  #define ZT_THREAD_ID (long)0
-#endif
-
 #if defined(__JNI_LIB__)
 		#include <jni.h>
 #endif
@@ -93,63 +85,50 @@
 		#define ZT_LOG_TAG "ZTSDK"
 #endif
 
-// Network stack debugging
-#if defined(__ANDROID__)
-	#define DEBUG_STACK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-			"STACK[%ld]: %17s:%5d:%20s: " fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-#elif defined(_WIN32)
-	#define DEBUG_STACK(fmt, ...) fprintf(stderr, ZT_YEL "STACK[%ld]: %17s:%5d:%25s: " fmt \
-		ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
-#else
-#define DEBUG_STACK(fmt, args ...) fprintf(stderr, ZT_YEL "STACK[%ld]: %17s:%5d:%25s: " fmt \
-		ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-#endif
-
-// Helpful file/line/function debugging macros
 #if defined(LIBZT_DEBUG) || defined(LIBZT_TRACE) || defined(__NATIVETEST__)
-	//
-	#if ZT_MSG_TEST == true
-		#if defined(__ANDROID__)
-			#define DEBUG_TEST(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-				"TEST : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-		#elif defined(_WIN32)
-			#define DEBUG_TEST(fmt, ...) fprintf(stderr, ZT_CYN "TEST [%ld]: %17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
-		#else
-		#define DEBUG_TEST(fmt, args ...) fprintf(stderr, ZT_CYN "TEST [%ld]: %17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		#endif
-	#else
-		#define DEBUG_TEST(fmt, args...)
-	#endif
-
 	//
 	#if ZT_MSG_ERROR == true
 		#if defined(__ANDROID__)
 			#define DEBUG_ERROR(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-				"ERROR: %17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+				"%17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
 		#elif defined(_WIN32)
-			#define DEBUG_ERROR(fmt, ...) fprintf(stderr, ZT_RED "ERROR[%ld]: %17s:%5d:%25s: " fmt "\n" \
+			#define DEBUG_ERROR(fmt, ...) fprintf(stderr, ZT_RED "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
 		#else
-		#define DEBUG_ERROR(fmt, args ...) fprintf(stderr, ZT_RED "ERROR[%ld]: %17s:%5d:%25s: " fmt "\n" \
+		#define DEBUG_ERROR(fmt, args ...) fprintf(stderr, ZT_RED "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
 		#endif
 	#else
 		#define DEBUG_ERROR(fmt, args...)
 	#endif
 
+	//
+	#if ZT_MSG_TEST == true
+		#if defined(__ANDROID__)
+			#define DEBUG_TEST(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"%17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#elif defined(_WIN32)
+			#define DEBUG_TEST(fmt, ...) fprintf(stderr, ZT_CYN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+		#else
+		#define DEBUG_TEST(fmt, args ...) fprintf(stderr, ZT_CYN "%17s:%5d:%25s: " fmt "\n" \
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_TEST(fmt, args...)
+	#endif
+
 	//
 	#if ZT_MSG_INFO == true
 		#if defined(__ANDROID__)
 			#define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-				"INFO : %17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+				"%17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
 		#elif defined(_WIN32)
-			#define DEBUG_INFO(fmt, ...) fprintf(stderr, ZT_WHT "INFO [%ld]: %17s:%5d:%25s: " fmt "\n" \
+			#define DEBUG_INFO(fmt, ...) fprintf(stderr, ZT_WHT "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
 		#else
-			#define DEBUG_INFO(fmt, args ...) fprintf(stderr, ZT_WHT "INFO [%ld]: %17s:%5d:%25s: " fmt "\n" \
+			#define DEBUG_INFO(fmt, args ...) fprintf(stderr, ZT_WHT "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
 		#endif
 	#else
 		#define DEBUG_INFO(fmt, args...)
@@ -159,52 +138,30 @@
 	#if ZT_MSG_TRANSFER == true
 		#if defined(__ANDROID__)
 			#define DEBUG_TRANS(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-				"TRANS: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+				"%17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
 		#elif defined(_WIN32)
-			#define DEBUG_TRANS(fmt, ...) fprintf(stderr, ZT_GRN "TRANS[%ld]: %17s:%5d:%25s: " fmt "\n" \
+			#define DEBUG_TRANS(fmt, ...) fprintf(stderr, ZT_GRN "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__)
 		#else
-			#define DEBUG_TRANS(fmt, args ...) fprintf(stderr, ZT_GRN "TRANS[%ld]: %17s:%5d:%25s: " fmt "\n" \
+			#define DEBUG_TRANS(fmt, args ...) fprintf(stderr, ZT_GRN "%17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+				ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
 		#endif
 	#else
 		#define DEBUG_TRANS(fmt, args...)
 	#endif
 
-	//
-	#if ZT_MSG_EXTRA == true
-		#if defined(__ANDROID__)
-			#define DEBUG_EXTRA(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-				"EXTRA: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-		#elif defined(_WIN32)
-			#define DEBUG_EXTRA(fmt, ...) fprintf(stderr, ZT_WHT "EXTRA[%ld]: %17s:%5d:%25s: " fmt "\n" \
-				ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, __VA_ARGS__, (long)0)
-		#else
-			#define DEBUG_EXTRA(fmt, args ...) fprintf(stderr, ZT_WHT "EXTRA[%ld]: %17s:%5d:%25s: " fmt "\n" \
-					ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		#endif
-	#else
-		#define DEBUG_EXTRA(fmt, args...)
-	#endif
-
 #else // !LIBZT_DEBUG || !__NATIVE_TEST__
 	#if defined(_WIN32)
 		#define DEBUG_ERROR(...)
 		#define DEBUG_TEST(...)
 		#define DEBUG_INFO(...)
-		#define DEBUG_BLANK(...)
-		#define DEBUG_ATTN(...)
 		#define DEBUG_TRANS(...)
-		#define DEBUG_EXTRA(...)
 	#else
 		#define DEBUG_ERROR(fmt, args...)
 		#define DEBUG_TEST(fmt, args...)
 		#define DEBUG_INFO(fmt, args...)
-		#define DEBUG_BLANK(fmt, args...)
-		#define DEBUG_ATTN(fmt, args...)
 		#define DEBUG_TRANS(fmt, args...)
-		#define DEBUG_EXTRA(fmt, args...)
 	#endif
 #endif
 
-#endif // _H
+#endif // _H

include/Defs.hpp

@@ -0,0 +1,211 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Management of virtual tap interfaces
+ */
+
+#ifndef LIBZT_DEFS_HPP
+#define LIBZT_DEFS_HPP
+
+#include "Constants.hpp"
+
+#ifndef _USING_LWIP_DEFINITIONS_
+#include <sys/socket.h>
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Subset of: ZeroTierOne.h                                                 //
+// We redefine a few ZT structures here so that we don't need to drag the   //
+// entire ZeroTierOne.h file into the user application                      //
+//////////////////////////////////////////////////////////////////////////////
+
+/**
+ * What trust hierarchy role does this peer have?
+ */
+enum zts_peer_role
+{
+	ZTS_PEER_ROLE_LEAF = 0,       // ordinary node
+	ZTS_PEER_ROLE_MOON = 1,       // moon root
+	ZTS_PEER_ROLE_PLANET = 2      // planetary root
+};
+
+/**
+ * A structure used to represent a virtual network route
+ */
+struct zts_virtual_network_route
+{
+	/**
+	 * Target network / netmask bits (in port field) or NULL or 0.0.0.0/0 for default
+	 */
+	struct sockaddr_storage target;
+
+	/**
+	 * Gateway IP address (port ignored) or NULL (family == 0) for LAN-local (no gateway)
+	 */
+	struct sockaddr_storage via;
+
+	/**
+	 * Route flags
+	 */
+	uint16_t flags;
+
+	/**
+	 * Route metric (not currently used)
+	 */
+	uint16_t metric;
+};
+
+/**
+ * A structure used to convey network-specific details to the user application
+ */
+struct zts_network_details
+{
+	/**
+	 * Network ID
+	 */
+	uint64_t nwid;
+
+	/**
+	 * Maximum Transmission Unit size for this network
+	 */
+	int mtu;
+
+	/**
+	 * Number of addresses (actually) assigned to the node on this network
+	 */
+	short num_addresses;
+
+	/**
+	 * Array of IPv4 and IPv6 addresses assigned to the node on this network
+	 */
+	struct sockaddr_storage addr[ZTS_MAX_ASSIGNED_ADDRESSES];
+
+	/**
+	 * Number of routes
+	 */
+	unsigned int num_routes;
+
+	/**
+	 * Array of IPv4 and IPv6 addresses assigned to the node on this network
+	 */
+	struct zts_virtual_network_route routes[ZTS_MAX_NETWORK_ROUTES];
+};
+
+/**
+ * Physical network path to a peer
+ */
+struct zts_physical_path
+{
+	/**
+	 * Address of endpoint
+	 */
+	struct sockaddr_storage address;
+
+	/**
+	 * Time of last send in milliseconds or 0 for never
+	 */
+	uint64_t lastSend;
+
+	/**
+	 * Time of last receive in milliseconds or 0 for never
+	 */
+	uint64_t lastReceive;
+
+	/**
+	 * Is this a trusted path? If so this will be its nonzero ID.
+	 */
+	uint64_t trustedPathId;
+
+	/**
+	 * Is path expired?
+	 */
+	int expired;
+
+	/**
+	 * Is path preferred?
+	 */
+	int preferred;
+};
+
+/**
+ * Peer status result buffer
+ */
+struct zts_peer_details
+{
+	/**
+	 * ZeroTier address (40 bits)
+	 */
+	uint64_t address;
+
+	/**
+	 * Remote major version or -1 if not known
+	 */
+	int versionMajor;
+
+	/**
+	 * Remote minor version or -1 if not known
+	 */
+	int versionMinor;
+
+	/**
+	 * Remote revision or -1 if not known
+	 */
+	int versionRev;
+
+	/**
+	 * Last measured latency in milliseconds or -1 if unknown
+	 */
+	int latency;
+
+	/**
+	 * What trust hierarchy role does this device have?
+	 */
+	enum zts_peer_role role;
+
+	/**
+	 * Number of paths (size of paths[])
+	 */
+	unsigned int pathCount;
+
+	/**
+	 * Known network paths to peer
+	 */
+	zts_physical_path paths[ZT_MAX_PEER_NETWORK_PATHS];
+};
+
+/**
+ * List of peers
+ */
+struct zts_peer_list
+{
+	zts_peer_details *peers;
+	unsigned long peerCount;
+};
+
+#endif // _H

include/RingBuffer.h

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -36,7 +36,6 @@
 #include <cstdint>
 #include <stdlib.h>
 
-
 typedef char bufElementType;
 
 class RingBuffer

include/ServiceControls.hpp

@@ -0,0 +1,373 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Header for ZeroTier service controls
+ */
+
+#ifndef LIBZT_SERVICE_CONTROLS_HPP
+#define LIBZT_SERVICE_CONTROLS_HPP
+
+#ifdef _WIN32
+	#ifdef ADD_EXPORTS
+		#define ZT_SOCKET_API __declspec(dllexport)
+	#else
+		#define ZT_SOCKET_API __declspec(dllimport)
+	#endif
+	#define ZTCALL __cdecl
+#else
+	#define ZT_SOCKET_API
+	#define ZTCALL
+#endif
+
+void api_sleep(int interval_ms);
+
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Service Controls                                                //
+//////////////////////////////////////////////////////////////////////////////
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief (optional) Sets the port for the background libzt service. If this function is called
+ * with a port number between 1-65535 it will attempt to bind to that port. If it is called with
+ * a port number of 0 it will attempt to randomly search for an available port. If this function
+ * is never called, the service will try to bind on LIBZT_DEFAULT_PORT which is 9994.
+ *
+ * @usage Should be called at the beginning of your application before `zts_startjoin()`
+ * @param portno Port number
+ * @return 0 if successful; or -1 if failed
+ */
+ZT_SOCKET_API int ZTCALL zts_set_service_port(int portno);
+
+/**
+ * @brief (optional) Returns the port number used by the ZeroTier service
+ * @usage Can be called if a port number was previously assigned
+ * @return the port number used by the ZeroTier service
+ */
+ZT_SOCKET_API int ZTCALL zts_get_service_port();
+
+/**
+ * @brief Starts the ZeroTier service
+ *
+ * @usage Should be called at the beginning of your application. Will blocks until all of the following conditions are met:
+ * - ZeroTier core service has been initialized
+ * - Cryptographic identity has been generated or loaded from directory specified by `path`
+ * - Virtual network is successfully joined
+ * - IP address is assigned by network controller service
+ * @param path path directory where cryptographic identities and network configuration files are stored and retrieved
+ *              (`identity.public`, `identity.secret`)
+ * @param blocking whether or not this call will block until the entire service is up and running
+ * @return 0 if successful; or 1 if failed
+ */
+ZT_SOCKET_API int ZTCALL zts_start(const char *path, int blocking);
+
+/**
+ * @brief Starts the ZeroTier service
+ *
+ * @usage Should be called at the beginning of your application. Will blocks until all of the following conditions are met:
+ * - ZeroTier core service has been initialized
+ * - Cryptographic identity has been generated or loaded from directory specified by `path`
+ * - Virtual network is successfully joined
+ * - IP address is assigned by network controller service
+ * @param path path directory where cryptographic identities and network configuration files are stored and retrieved
+ *              (`identity.public`, `identity.secret`)
+ * @param nwid A 16-digit hexidecimal network identifier (e.g. Earth: `8056c2e21c000001`)
+ * @return 0 if successful; or 1 if failed
+ */
+ZT_SOCKET_API int ZTCALL zts_startjoin(const char *path, const uint64_t nwid);
+
+/**
+ * @brief Stops the ZeroTier service, brings down all virtual interfaces in order to stop all traffic processing.
+ *
+ * @usage This should be called when the application anticipates not needing any sort of traffic processing for a
+ * prolonged period of time. The stack driver (with associated timers) will remain active in case future traffic
+ * processing is required. Note that the application must tolerate a multi-second startup time if zts_start()
+ * zts_startjoin() is called again. To stop this background thread and free all resources use zts_free() instead.
+ * @param blocking whether or not this call will block until the entire service is torn down
+ * @return Returns 0 on success, -1 on failure
+ */
+ZT_SOCKET_API int ZTCALL zts_stop(int blocking = 1);
+
+/**
+ * @brief Stops all background services, brings down all interfaces, frees all resources. After calling this function
+ * an application restart will be required before the library can be used again. This is a blocking call.
+ *
+ * @usage This should be called at the end of your program or when you do not anticipate communicating over ZeroTier
+ * @return Returns 0 on success, -1 on failure
+ */
+ZT_SOCKET_API int ZTCALL zts_free();
+
+/**
+ * @brief Return whether the ZeroTier service is currently running
+ *
+ * @usage Call this after zts_start()
+ * @return 1 if running, 0 if not running
+ */
+ZT_SOCKET_API int ZTCALL zts_core_running();
+
+/**
+ * @brief Return whether libzt is ready to handle socket API calls. Alternatively you could 
+ * have just called zts_startjoin(path, nwid)
+ *
+ * @usage Call this after zts_start()
+ * @return 1 if running, 0 if not running
+ */
+ZT_SOCKET_API int ZTCALL zts_ready();
+
+/**
+ * @brief Return the number of networks currently joined by this node
+ *
+ * @usage Call this after zts_start(), zts_startjoin() and/or zts_join()
+ * @return Number of networks joined by this node
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_get_num_joined_networks();
+
+/**
+ * @brief Populates a structure with details for a given network
+ *
+ * @usage Call this from the application thread any time after the node has joined a network
+ * @param nwid A 16-digit hexidecimal virtual network ID
+ * @param nd Pointer to a zts_network_details structure to populate
+ * @return ZTS_ERR_SERVICE if failed, 0 if otherwise
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_get_network_details(uint64_t nwid, struct zts_network_details *nd);
+
+/**
+ * @brief Populates an array of structures with details for any given number of networks
+ *
+ * @usage Call this from the application thread any time after the node has joined a network
+ * @param nds Pointer to an array of zts_network_details structures to populate
+ * @param num Number of zts_network_details structures available to copy data into, will be updated
+ * to reflect number of structures that were actually populated
+ * @return ZTS_ERR_SERVICE if failed, 0 if otherwise
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_get_all_network_details(struct zts_network_details *nds, int *num);
+
+/**
+ * @brief Join a network
+ *
+ * @usage Call this from application thread. Only after zts_start() has succeeded
+ * @param nwid A 16-digit hexidecimal virtual network ID
+ * @return 0 if successful, -1 for any failure
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_join(const uint64_t nwid, int blocking = 1);
+
+/**
+ * @brief Leave a network
+ *
+ * @usage Call this from application thread. Only after zts_start() has succeeded
+ * @param nwid A 16-digit hexidecimal virtual network ID
+ * @return 0 if successful, -1 for any failure
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_leave(const uint64_t nwid, int blocking = 1);
+
+
+/**
+ * @brief Leaves all networks
+ *
+ * @usage Call this from application thread. Only after zts_start() has succeeded
+ * @param nwid A 16-digit hexidecimal virtual network ID
+ * @return 0 if successful, -1 for any failure
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_leave_all(int blocking = 1);
+
+/**
+ * @brief Orbits a given moon (user-defined root server)
+ *
+ * @usage Call this from application thread. Only after zts_start() has succeeded
+ * @param moonWorldId A 16-digit hexidecimal world ID
+ * @param moonSeed A 16-digit hexidecimal seed ID
+ * @return ZTS_ERR_OK if successful, ZTS_ERR_SERVICE, ZTS_ERR_INVALID_ARG, ZTS_ERR_INVALID_OP if otherwise
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_orbit(uint64_t moonWorldId, uint64_t moonSeed);
+
+/**
+ * @brief De-orbits a given moon (user-defined root server)
+ *
+ * @usage Call this from application thread. Only after zts_start() has succeeded
+ * @param moonWorldId A 16-digit hexidecimal world ID
+ * @return ZTS_ERR_OK if successful, ZTS_ERR_SERVICE, ZTS_ERR_INVALID_ARG, ZTS_ERR_INVALID_OP if otherwise
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_deorbit(uint64_t moonWorldId);
+
+/**
+ * @brief Copies the configuration path used by ZeroTier into the provided buffer
+ *
+ * @usage Use this to determine where ZeroTier is storing identity files
+ * @param homePath Path to ZeroTier configuration files
+ * @param len Length of destination buffer
+ * @return 0 if no error, -1 if invalid argument was supplied
+ */
+ZT_SOCKET_API zts_err_t ZTCALL zts_get_path(char *homePath, size_t *len);
+
+/**
+ * @brief Returns the node ID of this instance
+ *
+ * @usage Call this after zts_start() and/or when zts_running() returns true
+ * @return
+ */
+ZT_SOCKET_API uint64_t ZTCALL zts_get_node_id();
+
+/**
+ * @brief Returns whether any address has been assigned to the SockTap for this network
+ *
+ * @usage This is used as an indicator of readiness for service for the ZeroTier core and stack
+ * @param nwid Network ID
+ * @return
+ */
+ZT_SOCKET_API int ZTCALL zts_has_address(const uint64_t nwid);
+
+
+/**
+ * @brief Returns the number of addresses assigned to this node for the given nwid
+ *
+ * @param nwid Network ID
+ * @return The number of addresses assigned
+ */
+ZT_SOCKET_API int ZTCALL zts_get_num_assigned_addresses(const uint64_t nwid);
+
+/**
+ * @brief Returns the assigned address located at the given index
+ *
+ * @usage The indices of each assigned address are not guaranteed and should only
+ * be used for iterative purposes.
+ * @param nwid Network ID
+ * @param index location of assigned address
+ * @return The number of addresses assigned
+ */
+ZT_SOCKET_API int ZTCALL zts_get_address_at_index(
+	const uint64_t nwid, const int index, struct sockaddr *addr, socklen_t *addrlen);
+
+/**
+ * @brief Get IP address for this device on a given network
+ *
+ * @usage FIXME: Only returns first address found, good enough for most cases
+ * @param nwid Network ID
+ * @param addr Destination structure for address
+ * @param addrlen size of destination address buffer, will be changed to size of returned address
+ * @return 0 if an address was successfully found, -1 if failure
+ */
+ZT_SOCKET_API int ZTCALL zts_get_address(
+	const uint64_t nwid, struct sockaddr_storage *addr, const int address_family);
+
+/**
+ * @brief Computes a 6PLANE IPv6 address for the given Network ID and Node ID
+ *
+ * @usage Can call any time
+ * @param addr Destination structure for address
+ * @param nwid Network ID 
+ * @param nodeId Node ID
+ * @return
+ */
+ZT_SOCKET_API void ZTCALL zts_get_6plane_addr(
+	struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId);
+
+/**
+ * @brief Computes a RFC4193 IPv6 address for the given Network ID and Node ID
+ *
+ * @usage Can call any time
+ * @param addr Destination structure for address
+ * @param nwid Network ID 
+ * @param nodeId Node ID
+ * @return
+ */
+ZT_SOCKET_API void ZTCALL zts_get_rfc4193_addr(
+	struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId);
+
+/**
+ * @brief Return the number of peers
+ *
+ * @usage Call this after zts_start() has succeeded
+ * @return
+ */
+ZT_SOCKET_API zts_err_t zts_get_peer_count();
+
+ZT_SOCKET_API zts_err_t zts_get_peers(struct zts_peer_details *pds, int *num);
+
+/**
+ * @brief Enables the HTTP backplane management system
+ *
+ * @usage Call this after zts_start() has succeeded
+ * @return ZTS_ERR_OK if successful, ZTS_ERR_SERVICE if otherwise
+ */
+ZT_SOCKET_API zts_err_t zts_enable_http_backplane_mgmt();
+
+/**
+ * @brief Disables the HTTP backplane management system
+ *
+ * @usage Call this after zts_start() has succeeded
+ * @return ZTS_ERR_OK if successful, ZTS_ERR_SERVICE, ZTS_ERR_INVALID_OP if otherwise
+ */
+ZT_SOCKET_API zts_err_t zts_disable_http_backplane_mgmt();
+
+
+/**
+ * @brief Starts a ZeroTier service in the background
+ *
+ * @usage For internal use only.
+ * @param
+ * @return
+ */
+#if defined(_WIN32)
+DWORD WINAPI _zts_start_service(LPVOID thread_id);
+#else
+void *_zts_start_service(void *thread_id);
+#endif
+
+/**
+ * @brief [Should not be called from user application] This function must be surrounded by 
+ * ZT service locks. It will determine if it is currently safe and allowed to operate on 
+ * the service.
+ * @usage Can be called at any time
+ * @return 1 or 0
+ */
+int _zts_can_perform_service_operation();
+
+/**
+ * @brief [Should not be called from user application] Returns whether or not the node is 
+ * online.
+ * @usage Can be called at any time
+ * @return 1 or 0
+ */
+int _zts_node_online();
+
+/**
+ * @brief [Should not be called from user application] Adjusts the delay multiplier for the
+ * network stack driver thread.
+ * @usage Can be called at any time
+ */
+void _hibernate_if_needed();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // _H

include/VirtualTap.hpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -30,24 +30,20 @@
  * Virtual Ethernet tap device
  */
 
-#ifndef ZT_VIRTUALTAP_H
+#ifndef LIBZT_VIRTUALTAP_HPP
-#define ZT_VIRTUALTAP_H
+#define LIBZT_VIRTUALTAP_HPP
 
 #ifndef _MSC_VER
 extern int errno;
 #endif
 
-#include "Mutex.hpp"
-#include "MulticastGroup.hpp"
-#include "InetAddress.hpp"
-#include "Thread.hpp"
 #include "Phy.hpp"
+#include "Thread.hpp"
+#include "InetAddress.hpp"
+#include "MulticastGroup.hpp"
+#include "Mutex.hpp"
 
-#include "libzt.h"
+#include "Defs.hpp"
-
-#include <vector>
-extern std::vector<void*> vtaps;
-extern ZeroTier::Mutex _vtaps_lock;
 
 #if defined(_WIN32)
 #include <WinSock2.h>
@@ -56,9 +52,9 @@ extern ZeroTier::Mutex _vtaps_lock;
 #include <Ifdef.h>
 #endif
 
-using namespace ZeroTier;
+namespace ZeroTier {
 
-class VirtualSocket;
+class Mutex;
 
 /**
  * emulates an Ethernet tap device
@@ -70,13 +66,13 @@ class VirtualTap
 public:
 	VirtualTap(
 		const char *homePath,
-		const ZeroTier::MAC &mac,
+		const MAC &mac,
 		unsigned int mtu,
 		unsigned int metric,
 		uint64_t nwid,
 		const char *friendlyName,
-		void (*handler)(void *, void *, uint64_t, const ZeroTier::MAC &,
+		void (*handler)(void *, void *, uint64_t, const MAC &,
-			const ZeroTier::MAC &, unsigned int, unsigned int, const void *, unsigned int),
+			const MAC &, unsigned int, unsigned int, const void *, unsigned int),
 		void *arg);
 
 	~VirtualTap();
@@ -87,27 +83,27 @@ public:
 	/**
 	 * Registers a device with the given address
 	 */
-	void registerIpWithStack(const ZeroTier::InetAddress &ip);
+	void registerIpWithStack(const InetAddress &ip);
 
 	/**
 	 * Adds an address to the userspace stack interface associated with this VirtualTap
 	 * - Starts VirtualTap main thread ONLY if successful
 	 */
-	bool addIp(const ZeroTier::InetAddress &ip);
+	bool addIp(const InetAddress &ip);
 
 	/**
 	 * Removes an address from the userspace stack interface associated with this VirtualTap
 	 */
-	bool removeIp(const ZeroTier::InetAddress &ip);
+	bool removeIp(const InetAddress &ip);
 
 	/**
 	 * Presents data to the userspace stack
 	 */
-	void put(const ZeroTier::MAC &from,const ZeroTier::MAC &to,unsigned int etherType,const void *data,
+	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,
 		unsigned int len);
 
 	/**
-	 * Get VirtualTap device name (e.g. 'libzt4-17d72843bc2c5760')
+	 * Get VirtualTap device name (e.g. 'libzt17d72843bc2c5760')
 	 */
 	std::string deviceName() const;
 
@@ -124,8 +120,8 @@ public:
 	/**
 	 * Scan multicast groups
 	 */
-	void scanMulticastGroups(std::vector<ZeroTier::MulticastGroup> &added,
+	void scanMulticastGroups(std::vector<MulticastGroup> &added,
-		std::vector<ZeroTier::MulticastGroup> &removed);
+		std::vector<MulticastGroup> &removed);
 
 	/**
 	 * Set MTU
@@ -155,49 +151,51 @@ public:
 	/**
 	 * For moving data onto the ZeroTier virtual wire
 	 */
-	void (*_handler)(void *, void *, uint64_t, const ZeroTier::MAC &, const ZeroTier::MAC &, unsigned int, unsigned int,
+	void (*_handler)(void *, void *, uint64_t, const MAC &, const MAC &, unsigned int, unsigned int,
 		const void *, unsigned int);
 
-	void phyOnUnixClose(ZeroTier::PhySocket *sock, void **uptr);
+	void phyOnUnixClose(PhySocket *sock, void **uptr);
-	void phyOnUnixData(ZeroTier::PhySocket *sock, void **uptr, void *data, ssize_t len);
+	void phyOnUnixData(PhySocket *sock, void **uptr, void *data, ssize_t len);
-	void phyOnUnixWritable(ZeroTier::PhySocket *sock, void **uptr, bool stack_invoked);
+	void phyOnUnixWritable(PhySocket *sock, void **uptr, bool stack_invoked);
 
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
-	/* Vars                                                                     */
+	// Vars                                                                     //
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
 
-	std::vector<std::pair<ZeroTier::InetAddress, ZeroTier::InetAddress> > routes;
+	std::vector<std::pair<InetAddress, InetAddress> > routes;
 	void *zt1ServiceRef = NULL;
 
 	char vtap_full_name[64];
 	char vtap_abbr_name[16];
 
-	static int devno;
 	size_t ifindex = 0;
 
-	std::vector<ZeroTier::InetAddress> ips() const;
+	std::vector<InetAddress> ips() const;
-	std::vector<ZeroTier::InetAddress> _ips;
+	std::vector<InetAddress> _ips;
 
 	std::string _homePath;
 	void *_arg;
 	volatile bool _initialized;
 	volatile bool _enabled;
 	volatile bool _run;
-	ZeroTier::MAC _mac;
+	MAC _mac;
 	unsigned int _mtu;
 	uint64_t _nwid;
-	ZeroTier::PhySocket *_unixListenSocket;
+	PhySocket *_unixListenSocket;
-	ZeroTier::Phy<VirtualTap *> _phy;
+	Phy<VirtualTap *> _phy;
-
-	std::vector<VirtualSocket*> _VirtualSockets;
 
 	Thread _thread;
+
+	int _shutdownSignalPipe[2];
+
 	std::string _dev; // path to Unix domain socket
 
 	std::vector<MulticastGroup> _multicastGroups;
 	Mutex _multicastGroups_m;
 	Mutex _ips_m, _tcpconns_m, _rx_buf_m, _close_m;
 
+	struct zts_network_details nd;
+
 	/*
 	 * Timestamp of last run of housekeeping
 	 * SEE: ZT_HOUSEKEEPING_INTERVAL in libzt.h
@@ -205,13 +203,13 @@ public:
 	uint64_t last_housekeeping_ts = 0;
 
 	/**
-	 * Disposes of previously-closed VirtualSockets
+	 * Performs miscellaneous background tasks
 	 */
 	void Housekeeping();
 
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
-	/* Not used in this implementation                                          */
+	// Not used in this implementation                                          //
-	/****************************************************************************/
+	//////////////////////////////////////////////////////////////////////////////
 
 	void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *local_address,
 		const struct sockaddr *from,void *data,unsigned long len);
@@ -223,5 +221,6 @@ public:
 	void phyOnTcpWritable(PhySocket *sock,void **uptr);
 };
 
+} // namespace ZeroTier
 
 #endif // _H

include/VirtualTapManager.hpp

@@ -0,0 +1,140 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * Management of virtual tap interfaces
+ */
+
+#include "VirtualTap.hpp"
+#include "OneService.hpp"
+
+namespace ZeroTier {
+
+extern std::vector<void*> vtaps;
+extern Mutex _vtaps_lock;
+
+class VirtualTap;
+
+/**
+ * @brief Static utility class for safely handling VirtualTap(s)
+ */
+class VirtualTapManager
+{
+public:
+
+	static void add_tap(VirtualTap *tap) {
+		_vtaps_lock.lock();
+		vtaps.push_back((void*)tap);
+		_vtaps_lock.unlock();
+	}
+
+	static VirtualTap *getTapByNWID(uint64_t nwid)
+	{
+		_vtaps_lock.lock();
+		VirtualTap *s, *tap = nullptr;
+		for (size_t i=0; i<vtaps.size(); i++) {
+			s = (VirtualTap*)vtaps[i];
+			if (s->_nwid == nwid) { tap = s; }
+		}
+		_vtaps_lock.unlock();
+		return tap;
+	}
+
+	static size_t get_vtaps_size() {
+		size_t sz;
+		_vtaps_lock.lock();
+		sz = vtaps.size();
+		_vtaps_lock.unlock();
+		return sz;
+	}
+
+	// TODO: We shouldn't re-apply the reference to everything all the time
+	static void update_service_references(void *serviceRef) {
+		_vtaps_lock.lock();
+		for (size_t i=0;i<vtaps.size(); i++) {
+			VirtualTap *s = (VirtualTap*)vtaps[i];
+			s->zt1ServiceRef=serviceRef;
+		}
+		_vtaps_lock.unlock();
+	}
+
+	static void remove_by_nwid(uint64_t nwid) {
+		_vtaps_lock.lock();
+		for (size_t i=0;i<vtaps.size(); i++) {
+			VirtualTap *s = (VirtualTap*)vtaps[i];
+			if (s->_nwid == nwid) {
+				vtaps.erase(vtaps.begin() + i);
+			}
+		}
+		_vtaps_lock.unlock();
+	}
+
+	static void clear() {
+		_vtaps_lock.lock();
+		vtaps.clear();
+		_vtaps_lock.unlock();
+	}
+
+	static void get_network_details(uint64_t nwid, struct zts_network_details *nd)
+	{
+		VirtualTap *tap;
+		socklen_t addrlen;
+		_vtaps_lock.lock();
+		for (size_t i=0; i<vtaps.size(); i++) {
+			tap = (VirtualTap*)vtaps[i];
+			if (tap->_nwid == nwid) {
+				nd->nwid = tap->_nwid;
+				nd->mtu = tap->_mtu;
+				// assigned addresses
+				nd->num_addresses = tap->_ips.size() < ZTS_MAX_ASSIGNED_ADDRESSES ? tap->_ips.size() : ZTS_MAX_ASSIGNED_ADDRESSES;
+				for (int j=0; j<nd->num_addresses; j++) {
+					addrlen = tap->_ips[j].isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+					memcpy(&(nd->addr[j]), &(tap->_ips[j]), addrlen);
+				}
+				// routes
+				nd->num_routes = ZTS_MAX_NETWORK_ROUTES;
+				OneService *zt1Service = (OneService*)tap->zt1ServiceRef;
+				zt1Service->getRoutes(nwid, (ZT_VirtualNetworkRoute*)&(nd->routes)[0], &(nd->num_routes));
+				break;
+			}
+		}
+		_vtaps_lock.unlock();
+	}
+
+	static void get_all_network_details(struct zts_network_details *nds, int *num)
+	{
+		VirtualTap *tap;
+		*num = get_vtaps_size();
+		for (size_t i=0; i<vtaps.size(); i++) {
+			tap = (VirtualTap*)vtaps[i];
+			get_network_details(tap->_nwid, &nds[i]);
+		}
+	}
+};
+
+} // namespace ZeroTier

include/Xcode-Bridging-Header.h

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -24,23 +24,34 @@
  * of your own application.
  */
 
+/**
+ * @file
+ *
+ * ZeroTier socket API
+ */
+
 #ifndef LIBZT_BRIDGING_HEADER_H
 #define LIBZT_BRIDGING_HEADER_H
 
 #include <sys/socket.h>
 #include "libzt.h"
 
-// ZT SERVICE CONTROLS
+//////////////////////////////////////////////////////////////////////////////
+// Service Controls                                                         //
+//////////////////////////////////////////////////////////////////////////////
+
 int zts_start(const char *path, int blocking);
 int zts_startjoin(const char *path, const uint64_t nwid);
 void zts_stop();
-int zts_core_running();
-int zts_stack_running();
 int zts_ready();
 int zts_join(uint64_t nwid);
 int zts_leave(uint64_t nwid);
 uint64_t zts_get_node_id();
-// SOCKET API
+
+//////////////////////////////////////////////////////////////////////////////
+// Socket API                                                               //
+//////////////////////////////////////////////////////////////////////////////
+
 int zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen);
 int zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen);
 int zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen);
@@ -64,7 +75,7 @@ int zts_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, s
 int zts_fcntl(int fd, int cmd, int flags);
 int zts_ioctl(int fd, unsigned long request, void *argp);
 
-#endif /* LIBZT_BRIDGING_HEADER_H */
+#endif // _H
 
 
 

include/libzt.h

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -27,33 +27,12 @@
 /**
  * @file
  *
- * Application-facing, partially-POSIX-compliant socket API
+ * ZeroTier socket API
  */
 
 #ifndef LIBZT_H
 #define LIBZT_H
 
-#include "libztDebug.h"
-#include "libztDefs.h"
-
-#include <stdlib.h>
-#include <stdint.h>
- 
-#if defined(__linux__) || defined(__APPLE__)
-#include <sys/socket.h>
-#include <unistd.h>
-#endif
-
-#if defined(_WIN32)
-#include <WinSock2.h>
-#include <stdint.h>
-#include <WS2tcpip.h>
-#endif
-
-/****************************************************************************/
-/* DLL export for Windows                                                   */
-/****************************************************************************/
-
 #ifdef _WIN32
 	#ifdef ADD_EXPORTS
 		#define ZT_SOCKET_API __declspec(dllexport)
@@ -66,211 +45,290 @@
 	#define ZTCALL
 #endif
 
-/****************************************************************************/
+#include <stdint.h>
-/* ZeroTier Service Controls                                                */
+#include <vector>
-/****************************************************************************/
+
+#if !defined(_WIN32) && !defined(__ANDROID__)
+typedef unsigned int socklen_t;
+//#include <sys/socket.h>
+#else
+typedef int socklen_t;
+//#include <sys/socket.h>
+#endif
+
+#if defined(_WIN32)
+#include <WinSock2.h>
+#include <stdint.h>
+#include <WS2tcpip.h>
+#include <Windows.h>
+#endif
+
+#ifdef _USING_LWIP_DEFINITIONS_
+#include "lwip/sockets.h"
+#endif
+
+#include "Constants.hpp"
+#include "Defs.hpp"
+#include "ServiceControls.hpp"
+
+class InetAddress;
+class VirtualTap;
+
+#if defined(_MSC_VER)
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Common definitions and structures for interacting with the ZT socket API //
+// This is a subset of lwip/sockets.h, lwip/arch.h, and lwip/inet.h         //
+//                                                                          //
+// These re-definitions exist here so that the user application's usage     //
+// of the API is internally consistent with the underlying network stack.   //
+// They have an attached prefix so that they can co-exist with the native   //
+// platform's own definitions and structures.                               // 
+//////////////////////////////////////////////////////////////////////////////
+
+// Socket protocol types
+#define ZTS_SOCK_STREAM     0x0001
+#define ZTS_SOCK_DGRAM      0x0002
+#define ZTS_SOCK_RAW        0x0003
+// Socket family types
+#define ZTS_AF_UNSPEC       0x0000
+#define ZTS_AF_INET         0x0002
+#define ZTS_AF_INET6        0x000a
+#define ZTS_PF_INET         ZTS_AF_INET
+#define ZTS_PF_INET6        ZTS_AF_INET6
+#define ZTS_PF_UNSPEC       ZTS_AF_UNSPEC
+// 
+#define ZTS_IPPROTO_IP      0x0000
+#define ZTS_IPPROTO_ICMP    0x0001
+#define ZTS_IPPROTO_TCP     0x0006
+#define ZTS_IPPROTO_UDP     0x0011
+#define ZTS_IPPROTO_IPV6    0x0029
+#define ZTS_IPPROTO_ICMPV6  0x003a
+#define ZTS_IPPROTO_UDPLITE 0x0088
+#define ZTS_IPPROTO_RAW     0x00ff
+// send() and recv() flags
+#define ZTS_MSG_PEEK        0x0001
+#define ZTS_MSG_WAITALL     0x0002 // NOT YET SUPPORTED
+#define ZTS_MSG_OOB         0x0004 // NOT YET SUPPORTED
+#define ZTS_MSG_DONTWAIT    0x0008
+#define ZTS_MSG_MORE        0x0010
+// fnctl() commands
+#define ZTS_F_GETFL         0x0003
+#define ZTS_F_SETFL         0x0004
+// fnctl() flags
+#define ZTS_O_NONBLOCK      0x0001
+#define ZTS_O_NDELAY        0x0001
+//
+#define ZTS_SHUT_RD         0x0000
+#define ZTS_SHUT_WR         0x0001
+#define ZTS_SHUT_RDWR       0x0002
+// Socket level option number
+#define ZTS_SOL_SOCKET      0x0fff
+// Socket options
+#define ZTS_SO_DEBUG        0x0001 // NOT YET SUPPORTED
+#define ZTS_SO_ACCEPTCONN   0x0002
+#define ZTS_SO_REUSEADDR    0x0004
+#define ZTS_SO_KEEPALIVE    0x0008
+#define ZTS_SO_DONTROUTE    0x0010 // NOT YET SUPPORTED
+#define ZTS_SO_BROADCAST    0x0020
+#define ZTS_SO_USELOOPBACK  0x0040 // NOT YET SUPPORTED
+#define ZTS_SO_LINGER       0x0080
+#define ZTS_SO_DONTLINGER   ((int)(~ZTS_SO_LINGER))
+#define ZTS_SO_OOBINLINE    0x0100 // NOT YET SUPPORTED
+#define ZTS_SO_REUSEPORT    0x0200 // NOT YET SUPPORTED
+#define ZTS_SO_SNDBUF       0x1001 // NOT YET SUPPORTED
+#define ZTS_SO_RCVBUF       0x1002
+#define ZTS_SO_SNDLOWAT     0x1003 // NOT YET SUPPORTED
+#define ZTS_SO_RCVLOWAT     0x1004 // NOT YET SUPPORTED
+#define ZTS_SO_SNDTIMEO     0x1005
+#define ZTS_SO_RCVTIMEO     0x1006
+#define ZTS_SO_ERROR        0x1007
+#define ZTS_SO_TYPE         0x1008
+#define ZTS_SO_CONTIMEO     0x1009
+#define ZTS_SO_NO_CHECK     0x100a
+// IPPROTO_IP options
+#define ZTS_IP_TOS          0x0001
+#define ZTS_IP_TTL          0x0002
+// IPPROTO_TCP options
+#define ZTS_TCP_NODELAY     0x0001
+#define ZTS_TCP_KEEPALIVE   0x0002
+#define ZTS_TCP_KEEPIDLE    0x0003
+#define ZTS_TCP_KEEPINTVL   0x0004
+#define ZTS_TCP_KEEPCNT     0x0005
+// IPPROTO_IPV6 options
+#define ZTS_IPV6_CHECKSUM   0x0007  // RFC3542
+#define ZTS_IPV6_V6ONLY     0x001b  // RFC3493
+//
+#define ZTS_IOCPARM_MASK    0x7fU
+#define ZTS_IOC_VOID        0x20000000UL
+#define ZTS_IOC_OUT         0x40000000UL
+#define ZTS_IOC_IN          0x80000000UL
+#define ZTS_IOC_INOUT       (ZTS_IOC_IN   | ZTS_IOC_OUT)
+#define ZTS_IO(x,y)         (ZTS_IOC_VOID | ((x)<<8)|(y))
+#define ZTS_IOR(x,y,t)      (ZTS_IOC_OUT  | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
+#define ZTS_IOW(x,y,t)      (ZTS_IOC_IN   | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
+//
+#define ZTS_FIONREAD        ZTS_IOR('f', 127, unsigned long)
+#define ZTS_FIONBIO         ZTS_IOW('f', 126, unsigned long)
+
+
+/* FD_SET used for lwip_select */
+
+#ifndef ZTS_FD_SET
+#undef  ZTS_FD_SETSIZE
+// Make FD_SETSIZE match NUM_SOCKETS in socket.c
+#define ZTS_FD_SETSIZE    MEMP_NUM_NETCONN
+#define ZTS_FDSETSAFESET(n, code) do { \
+  if (((n) - LWIP_SOCKET_OFFSET < MEMP_NUM_NETCONN) && (((int)(n) - LWIP_SOCKET_OFFSET) >= 0)) { \
+  code; }} while(0)
+#define ZTS_FDSETSAFEGET(n, code) (((n) - LWIP_SOCKET_OFFSET < MEMP_NUM_NETCONN) && (((int)(n) - LWIP_SOCKET_OFFSET) >= 0) ?\
+  (code) : 0)
+#define ZTS_FD_SET(n, p)  ZTS_FDSETSAFESET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] |=  (1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
+#define ZTS_FD_CLR(n, p)  ZTS_FDSETSAFESET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] &= ~(1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
+#define ZTS_FD_ISSET(n,p) ZTS_FDSETSAFEGET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] &   (1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
+#define ZTS_FD_ZERO(p)    memset((void*)(p), 0, sizeof(*(p)))
+
+#elif LWIP_SOCKET_OFFSET
+#error LWIP_SOCKET_OFFSET does not work with external FD_SET!
+#elif ZTS_FD_SETSIZE < MEMP_NUM_NETCONN
+#error "external ZTS_FD_SETSIZE too small for number of sockets"
+#endif // FD_SET
+
+#if !defined(_USING_LWIP_DEFINITIONS_)
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/**
+typedef uint8_t   u8_t;
- * @brief (optional) Sets the port for the background libzt service. If this function is called
+typedef int8_t    s8_t;
- * with a port number between 1-65535 it will attempt to bind to that port. If it is called with
+typedef uint16_t  u16_t;
- * a port number of 0 it will attempt to randomly search for an available port. If this function
+typedef int16_t   s16_t;
- * is never called, the service will try to bind on LIBZT_DEFAULT_PORT which is 9994.
+typedef uint32_t  u32_t;
- *
+typedef int32_t   s32_t;
- * @usage Should be called at the beginning of your application before `zts_startjoin()`
+typedef uintptr_t mem_ptr_t;
- * @param portno Port number
+
- * @return 0 if successful; or -1 if failed
+typedef u32_t zts_in_addr_t;
+typedef u16_t zts_in_port_t;
+typedef u8_t zts_sa_family_t;
+
+struct zts_in_addr {
+  zts_in_addr_t s_addr;
+};
+
+struct zts_in6_addr {
+  union {
+    u32_t u32_addr[4];
+    u8_t  u8_addr[16];
+  } un;
+#define s6_addr  un.u8_addr
+};
+
+struct zts_sockaddr_in {
+  u8_t            sin_len;
+  zts_sa_family_t     sin_family;
+  zts_in_port_t       sin_port;
+  struct zts_in_addr  sin_addr;
+#define SIN_ZERO_LEN 8
+  char            sin_zero[SIN_ZERO_LEN];
+};
+
+struct zts_sockaddr_in6 {
+  u8_t            sin6_len;      /* length of this structure    */
+  zts_sa_family_t     sin6_family;   /* AF_INET6                    */
+  zts_in_port_t       sin6_port;     /* Transport layer port #      */
+  u32_t           sin6_flowinfo; /* IPv6 flow information       */
+  struct zts_in6_addr sin6_addr;     /* IPv6 address                */
+  u32_t           sin6_scope_id; /* Set of interfaces for scope */
+};
+
+struct zts_sockaddr {
+  u8_t        sa_len;
+  zts_sa_family_t sa_family;
+  char        sa_data[14];
+};
+
+struct zts_sockaddr_storage {
+  u8_t        s2_len;
+  zts_sa_family_t ss_family;
+  char        s2_data1[2];
+  u32_t       s2_data2[3];
+  u32_t       s2_data3[3];
+};
+
+#if !defined(zts_iovec)
+struct zts_iovec {
+  void  *iov_base;
+  size_t iov_len;
+};
+#endif
+
+struct zts_msghdr {
+  void         *msg_name;
+  socklen_t     msg_namelen;
+  struct iovec *msg_iov;
+  int           msg_iovlen;
+  void         *msg_control;
+  socklen_t     msg_controllen;
+  int           msg_flags;
+};
+
+/*
+ * Structure used for manipulating linger option.
  */
-ZT_SOCKET_API int ZTCALL zts_set_service_port(int portno);
+struct zts_linger {
+       int l_onoff;                /* option on/off */
+       int l_linger;               /* linger time in seconds */
+};
 
-/**
+/*
- * @brief Starts libzt
+typedef struct fd_set
- *
+{
- * @usage Should be called at the beginning of your application. Will blocks until all of the following conditions are met:
+  unsigned char fd_bits [(FD_SETSIZE+7)/8];
- * - ZeroTier core service has been initialized
+} fd_set;
- * - Cryptographic identity has been generated or loaded from directory specified by `path`
+*/
- * - Virtual network is successfully joined
- * - IP address is assigned by network controller service
- * @param path path directory where cryptographic identities and network configuration files are stored and retrieved
- *              (`identity.public`, `identity.secret`)
- * @param nwid A 16-digit hexidecimal network identifier (e.g. Earth: `8056c2e21c000001`)
- * @return 0 if successful; or 1 if failed
- */
-ZT_SOCKET_API int ZTCALL zts_start(const char *path, int blocking);
 
-/**
+#ifdef __cplusplus
- * @brief Starts libzt
+}
- *
+#endif
- * @usage Should be called at the beginning of your application. Will blocks until all of the following conditions are met:
- * - ZeroTier core service has been initialized
- * - Cryptographic identity has been generated or loaded from directory specified by `path`
- * - Virtual network is successfully joined
- * - IP address is assigned by network controller service
- * @param path path directory where cryptographic identities and network configuration files are stored and retrieved
- *              (`identity.public`, `identity.secret`)
- * @param nwid A 16-digit hexidecimal network identifier (e.g. Earth: `8056c2e21c000001`)
- * @return 0 if successful; or 1 if failed
- */
-ZT_SOCKET_API int ZTCALL zts_startjoin(const char *path, const uint64_t nwid);
 
-/**
+#endif // _USING_LWIP_DEFINITIONS_
- * @brief Stops ZeroTier core services, stack drivers, stack threads, etc
- *
- * @usage This should be called at the end of your program or when you do not anticipate communicating over ZeroTier
- * @return Returns 0 on success, -1 on failure
- */
-ZT_SOCKET_API void ZTCALL zts_stop();
 
-/**
+//////////////////////////////////////////////////////////////////////////////
- * @brief Return whether ZeroTier core service is currently running
+// For SOCK_RAW support, it will initially be modeled after linux's API, so //
- *
+// below are the various things we need to define in order to make this API //
- * @usage Call this after zts_start()
+// work on other platforms. Maybe later down the road we will customize	    //
- * @return 1 if running, 0 if not running
+// this for each different platform. Maybe.									//
- */
+//////////////////////////////////////////////////////////////////////////////
-ZT_SOCKET_API int ZTCALL zts_core_running();
 
-/**
+#if !defined(__linux__)
- * @brief Return whether the userspace network stack is currently running
+#define SIOCGIFINDEX 101
- *
+#define SIOCGIFHWADDR 102
- * @usage Call this after zts_start()
- * @return 1 if running, 0 if not running
- */
-ZT_SOCKET_API int ZTCALL zts_stack_running();
 
-/**
+// Normally defined in linux/if_packet.h, defined here so we can offer a linux-like
- * @brief Return whether libzt is ready to handle socket API calls. Alternatively you could 
+// raw socket API on non-linux platforms
- * have just called zts_startjoin(path, nwid)
+struct sockaddr_ll {
- *
+		unsigned short sll_family;	/* Always AF_PACKET */
- * @usage Call this after zts_start()
+		unsigned short sll_protocol; /* Physical layer protocol */
- * @return 1 if running, 0 if not running
+		int	sll_ifindex;  /* Interface number */
- */
+		unsigned short sll_hatype;	/* ARP hardware type */
-ZT_SOCKET_API int ZTCALL zts_ready();
+		unsigned char sll_pkttype;  /* Packet type */
+		unsigned char sll_halen;	 /* Length of address */
+		unsigned char sll_addr[8];  /* Physical layer address */
+};
 
-/**
+#endif
- * @brief Join a network
- *
- * @usage Call this from application thread. Only after zts_start() has succeeded
- * @param nwid A 16-digit hexidecimal virtual network ID
- * @return 0 if successful, -1 for any failure
- */
-ZT_SOCKET_API int ZTCALL zts_join(const uint64_t nwid);
 
-/**
+//////////////////////////////////////////////////////////////////////////////
- * @brief Leave a network
+// Socket API                                                               //
- *
+//////////////////////////////////////////////////////////////////////////////
- * @usage Call this from application thread. Only after zts_start() has succeeded
- * @param nwid A 16-digit hexidecimal virtual network ID
- * @return 0 if successful, -1 for any failure
- */
-ZT_SOCKET_API int ZTCALL zts_leave(const uint64_t nwid);
 
-/**
+#ifdef __cplusplus
- * @brief Copies the configuration path used by ZeroTier into the provided buffer
+extern "C" {
- *
+#endif
- * @usage
- * @param homePath Path to ZeroTier configuration files
- * @param len Length of destination buffer
- * @return 
- */
-ZT_SOCKET_API void ZTCALL zts_get_path(char *homePath, const size_t len);
-
-/**
- * @brief Returns the node ID of this instance
- *
- * @usage Call this after zts_start() and/or when zts_running() returns true
- * @return
- */
-ZT_SOCKET_API uint64_t ZTCALL zts_get_node_id();
-
-/**
- * @brief Returns the node ID of this instance (as read from a file)
- *
- * @usage Call with or without starting the service with zts_start()
- * @param filepath Path to ZeroTier configuration files
- * @return
- */
-ZT_SOCKET_API uint64_t ZTCALL zts_get_node_id_from_file(const char *filepath);
-
-/**
- * @brief Returns whether any address has been assigned to the SockTap for this network
- *
- * @usage This is used as an indicator of readiness for service for the ZeroTier core and stack
- * @param nwid Network ID
- * @return
- */
-ZT_SOCKET_API int ZTCALL zts_has_address(const uint64_t nwid);
-
-
-/**
- * @brief Returns the number of addresses assigned to this node for the given nwid
- *
- * @param nwid Network ID
- * @return The number of addresses assigned
- */
-ZT_SOCKET_API int ZTCALL zts_get_num_assigned_addresses(const uint64_t nwid);
-
-/**
- * @brief Returns the assigned address located at the given index
- *
- * @usage The indices of each assigned address are not guaranteed and should only
- * be used for iterative purposes.
- * @param nwid Network ID
- * @param index location of assigned address
- * @return The number of addresses assigned
- */
-ZT_SOCKET_API int ZTCALL zts_get_address_at_index(
-	const uint64_t nwid, const int index, struct sockaddr *addr, socklen_t *addrlen);
-
-/**
- * @brief Get IP address for this device on a given network
- *
- * @usage FIXME: Only returns first address found, good enough for most cases
- * @param nwid Network ID
- * @param addr Destination structure for address
- * @param addrlen size of destination address buffer, will be changed to size of returned address
- * @return 0 if an address was successfully found, -1 if failure
- */
-ZT_SOCKET_API int ZTCALL zts_get_address(
-	const uint64_t nwid, struct sockaddr_storage *addr, const int address_family);
-
-/**
- * @brief Computes a 6PLANE IPv6 address for the given Network ID and Node ID
- *
- * @usage Can call any time
- * @param addr Destination structure for address
- * @param nwid Network ID 
- * @param nodeId Node ID
- * @return
- */
-ZT_SOCKET_API void ZTCALL zts_get_6plane_addr(
-	struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId);
-
-/**
- * @brief Computes a RFC4193 IPv6 address for the given Network ID and Node ID
- *
- * @usage Can call any time
- * @param addr Destination structure for address
- * @param nwid Network ID 
- * @param nodeId Node ID
- * @return
- */
-ZT_SOCKET_API void ZTCALL zts_get_rfc4193_addr(
-	struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId);
-
-/**
- * @brief Return the number of peers
- *
- * @usage Call this after zts_start() has succeeded
- * @return
- */
-ZT_SOCKET_API unsigned long zts_get_peer_count();
-
-/****************************************************************************/
-/* Socket-like API                                                          */
-/****************************************************************************/
 
 /**
  * @brief Create a socket
@@ -286,7 +344,7 @@ ZT_SOCKET_API unsigned long zts_get_peer_count();
  * @param protocol Protocols supported on this socket
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_socket(int socket_family, int socket_type, int protocol);
+ZT_SOCKET_API zts_err_t ZTCALL zts_socket(int socket_family, int socket_type, int protocol);
 
 /**
  * @brief Connect a socket to a remote host
@@ -297,7 +355,7 @@ ZT_SOCKET_API int ZTCALL zts_socket(int socket_family, int socket_type, int prot
  * @param addrlen Length of address
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen);
+ZT_SOCKET_API zts_err_t ZTCALL zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen);
 
 /**
  * @brief Bind a socket to a virtual interface
@@ -308,7 +366,7 @@ ZT_SOCKET_API int ZTCALL zts_connect(int fd, const struct sockaddr *addr, sockle
  * @param addrlen Length of address
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen);
+ZT_SOCKET_API zts_err_t ZTCALL zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen);
 
 /**
  * @brief Listen for incoming connections
@@ -318,7 +376,7 @@ ZT_SOCKET_API int ZTCALL zts_bind(int fd, const struct sockaddr *addr, socklen_t
  * @param backlog Number of backlogged connection allowed
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_listen(int fd, int backlog);
+ZT_SOCKET_API zts_err_t ZTCALL zts_listen(int fd, int backlog);
 
 /**
  * @brief Accept an incoming connection
@@ -329,7 +387,7 @@ ZT_SOCKET_API int ZTCALL zts_listen(int fd, int backlog);
  * @param addrlen Length of address
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen);
+ZT_SOCKET_API zts_err_t ZTCALL zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen);
 
 /**
  * @brief Accept an incoming connection
@@ -342,7 +400,7 @@ ZT_SOCKET_API int ZTCALL zts_accept(int fd, struct sockaddr *addr, socklen_t *ad
  * @return
  */
 #if defined(__linux__)
-	int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags);
+	zts_err_t zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags);
 #endif
 
 /**
@@ -356,7 +414,7 @@ ZT_SOCKET_API int ZTCALL zts_accept(int fd, struct sockaddr *addr, socklen_t *ad
  * @param optlen Length of option value
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_setsockopt(
+ZT_SOCKET_API zts_err_t ZTCALL zts_setsockopt(
 	int fd, int level, int optname, const void *optval, socklen_t optlen);
 
 /**
@@ -370,7 +428,7 @@ ZT_SOCKET_API int ZTCALL zts_setsockopt(
  * @param optlen Length of value
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_getsockopt(
+ZT_SOCKET_API zts_err_t ZTCALL zts_getsockopt(
 	int fd, int level, int optname, void *optval, socklen_t *optlen);
 
 /**
@@ -382,7 +440,7 @@ ZT_SOCKET_API int ZTCALL zts_getsockopt(
  * @param addrlen Length of name
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen);
+ZT_SOCKET_API zts_err_t ZTCALL zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen);
 
 /**
  * @brief Get the peer name for the remote end of a connected socket
@@ -393,7 +451,7 @@ ZT_SOCKET_API int ZTCALL zts_getsockname(int fd, struct sockaddr *addr, socklen_
  * @param addrlen Length of name
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen);
+ZT_SOCKET_API zts_err_t ZTCALL zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen);
 
 /**
  * @brief Gets current hostname
@@ -403,7 +461,7 @@ ZT_SOCKET_API int ZTCALL zts_getpeername(int fd, struct sockaddr *addr, socklen_
  * @param len
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_gethostname(char *name, size_t len);
+ZT_SOCKET_API zts_err_t ZTCALL zts_gethostname(char *name, size_t len);
 
 /**
  * @brief Sets current hostname
@@ -413,7 +471,7 @@ ZT_SOCKET_API int ZTCALL zts_gethostname(char *name, size_t len);
  * @param len
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_sethostname(const char *name, size_t len);
+ZT_SOCKET_API zts_err_t ZTCALL zts_sethostname(const char *name, size_t len);
 
 /**
  * @brief Return a pointer to an object with the following structure describing an internet host referenced by name
@@ -431,7 +489,7 @@ ZT_SOCKET_API struct hostent *zts_gethostbyname(const char *name);
  * @param fd File descriptor (only valid for use with libzt calls)
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_close(int fd);
+ZT_SOCKET_API zts_err_t ZTCALL zts_close(int fd);
 
 /**
  * @brief Waits for one of a set of file descriptors to become ready to perform I/O.
@@ -460,7 +518,7 @@ int zts_poll(struct pollfd *fds, nfds_t nfds, int timeout);
  * @param timeout
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_select(
+ZT_SOCKET_API zts_err_t ZTCALL zts_select(
 	int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout);
 
 /**
@@ -476,7 +534,7 @@ ZT_SOCKET_API int ZTCALL zts_select(
 #define F_SETFL 0
 #define O_NONBLOCK 0
 #endif
-ZT_SOCKET_API int ZTCALL zts_fcntl(int fd, int cmd, int flags);
+ZT_SOCKET_API zts_err_t ZTCALL zts_fcntl(int fd, int cmd, int flags);
 
 /**
  * @brief Control a device
@@ -487,7 +545,7 @@ ZT_SOCKET_API int ZTCALL zts_fcntl(int fd, int cmd, int flags);
  * @param argp
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_ioctl(int fd, unsigned long request, void *argp);
+ZT_SOCKET_API zts_err_t ZTCALL zts_ioctl(int fd, unsigned long request, void *argp);
 
 /**
  * @brief Send data to remote host
@@ -574,7 +632,7 @@ ZT_SOCKET_API ssize_t ZTCALL zts_recvmsg(int fd, struct msghdr *msg,int flags);
  * @param len Length of data buffer to receive data
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_read(int fd, void *buf, size_t len);
+ZT_SOCKET_API zts_err_t ZTCALL zts_read(int fd, void *buf, size_t len);
 
 /**
  * @brief Write bytes from buffer to socket
@@ -585,7 +643,7 @@ ZT_SOCKET_API int ZTCALL zts_read(int fd, void *buf, size_t len);
  * @param len Length of buffer to write
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_write(int fd, const void *buf, size_t len);
+ZT_SOCKET_API zts_err_t ZTCALL zts_write(int fd, const void *buf, size_t len);
 
 /**
  * @brief Shut down some aspect of a socket (read, write, or both)
@@ -595,7 +653,7 @@ ZT_SOCKET_API int ZTCALL zts_write(int fd, const void *buf, size_t len);
  * @param how Which aspects of the socket should be shut down
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_shutdown(int fd, int how);
+ZT_SOCKET_API zts_err_t ZTCALL zts_shutdown(int fd, int how);
 
 /**
  * @brief Adds a DNS nameserver for the network stack to use
@@ -604,7 +662,7 @@ ZT_SOCKET_API int ZTCALL zts_shutdown(int fd, int how);
  * @param addr Address for DNS nameserver
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_add_dns_nameserver(struct sockaddr *addr);
+ZT_SOCKET_API zts_err_t ZTCALL zts_add_dns_nameserver(struct sockaddr *addr);
 
 /**
  * @brief Removes a DNS nameserver
@@ -613,7 +671,7 @@ ZT_SOCKET_API int ZTCALL zts_add_dns_nameserver(struct sockaddr *addr);
  * @param addr Address for DNS nameserver
  * @return
  */
-ZT_SOCKET_API int ZTCALL zts_del_dns_nameserver(struct sockaddr *addr);
+ZT_SOCKET_API zts_err_t ZTCALL zts_del_dns_nameserver(struct sockaddr *addr);
 
 #ifdef __cplusplus
 } // extern "C"

include/libztDefs.h

@@ -1,307 +0,0 @@
-/*
- * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * You can be released from the requirements of the license by purchasing
- * a commercial license. Buying such a license is mandatory as soon as you
- * develop commercial closed-source software that incorporates or links
- * directly against ZeroTier software without disclosing the source code
- * of your own application.
- */
-
-/**
- * @file
- *
- * Application-facing, partially-POSIX-compliant socket API
- */
-
-#ifndef LIBZT_DEFINES_H
-#define LIBZT_DEFINES_H
-
-#define LIBZT_IPV4 1
-#define LIBZT_IPV6 1
-
-/**
- * Default port that libzt will use to support all virtual communication
- */
-#define LIBZT_DEFAULT_PORT 9994
-
-#define NO_STACK   0 // for layer-2 only (this will omit all userspace network stack code)
-
-/**
- * Maximum MTU size for ZeroTier
- */
-#define ZT_MAX_MTU	10000
-
-/**
- * How fast service states are re-checked (in milliseconds)
- */
-#define ZTO_WRAPPER_CHECK_INTERVAL	100
-
-/**
- * Length of buffer required to hold a ztAddress/nodeID
- */
-#define ZTO_ID_LEN	16
-
-#if defined(_MSC_VER)
-#include <BaseTsd.h>
-typedef SSIZE_T ssize_t;
-#endif
-
-/****************************************************************************/
-/* For SOCK_RAW support, it will initially be modeled after linux's API, so */
-/* below are the various things we need to define in order to make this API */
-/* work on other platforms. Mayber later down the road we will customize	 */
-/* this for each different platform. Maybe.											*/
-/****************************************************************************/
-
-#if !defined(__linux__)
-#define SIOCGIFINDEX  101
-#define SIOCGIFHWADDR 102
-
-// Normally defined in linux/if_packet.h, defined here so we can offer a linux-like
-// raw socket API on non-linux platforms
-struct sockaddr_ll {
-		unsigned short	sll_family;	/* Always AF_PACKET */
-		unsigned short	sll_protocol; /* Physical layer protocol */
-		int				sll_ifindex;  /* Interface number */
-		unsigned short	sll_hatype;	/* ARP hardware type */
-		unsigned char	sll_pkttype;  /* Packet type */
-		unsigned char	sll_halen;	 /* Length of address */
-		unsigned char	sll_addr[8];  /* Physical layer address */
-};
-
-#endif
-
-/****************************************************************************/
-/* lwIP                                                                     */
-/****************************************************************************/
-
-// For LWIP configuration see: include/lwipopts.h
-
-/* The following three quantities are related and govern how incoming frames are fed into the 
-network stack's core:
-
-Every LWIP_GUARDED_BUF_CHECK_INTERVAL milliseconds, a callback will be called from the core and 
-will input a maximum of LWIP_FRAMES_HANDLED_PER_CORE_CALL frames before returning control back
-to the core. Meanwhile, incoming frames from the ZeroTier wire will be allocated and their 
-pointers will be cached in the receive frame buffer of the size LWIP_MAX_GUARDED_RX_BUF_SZ to 
-await the next callback from the core */
-
-#define LWIP_GUARDED_BUF_CHECK_INTERVAL	50 // in ms
-#define LWIP_MAX_GUARDED_RX_BUF_SZ	1024 // number of frame pointers that can be cached waiting for receipt into core
-#define LWIP_FRAMES_HANDLED_PER_CORE_CALL 16 // How many frames are handled per call from core
-
-typedef signed char err_t;
-
-#define ND6_DISCOVERY_INTERVAL 1000
-#define ARP_DISCOVERY_INTERVAL ARP_TMR_INTERVAL
-
-/**
- * Specifies the polling interval and the callback function that should
- * be called to poll the application. The interval is specified in
- * number of TCP coarse grained timer shots, which typically occurs
- * twice a second. An interval of 10 means that the application would
- * be polled every 5 seconds. (only for raw lwIP driver)
- */
-#define LWIP_APPLICATION_POLL_FREQ 2
-
-/**
- * TCP timer interval in milliseconds (only for raw lwIP driver)
- */
-#define LWIP_TCP_TIMER_INTERVAL 25
-
-/**
- * How often we check VirtualSocket statuses in milliseconds (only for raw lwIP driver)
- */
-#define LWIP_STATUS_TMR_INTERVAL 500
-
-// #define LWIP_CHKSUM <your_checksum_routine>, See: RFC1071 for inspiration
-
-/****************************************************************************/
-/* Defines                                                                  */
-/****************************************************************************/
-
-/**
- * Maximum number of sockets that libzt can administer
- */
-#define ZT_MAX_SOCKETS 1024
-
-/**
- * Maximum MTU size for libzt (must be less than or equal to ZT_MAX_MTU)
- */
-#define ZT_SDK_MTU ZT_MAX_MTU
-
-/**
- *
- */
-#define ZT_LEN_SZ 4
-
-/**
- *
- */
-#define ZT_ADDR_SZ 128
-
-/**
- * Size of message buffer for VirtualSockets
- */
-#define ZT_SOCKET_MSG_BUF_SZ ZT_SDK_MTU + ZT_LEN_SZ + ZT_ADDR_SZ
-
-/**
- * Polling interval (in ms) for file descriptors wrapped in the Phy I/O loop (for raw drivers only)
- */
-#define ZT_PHY_POLL_INTERVAL 5
-
-/**
- * State check interval (in ms) for VirtualSocket state
- */
-#define ZT_ACCEPT_RECHECK_DELAY 50
-
-/**
- * State check interval (in ms) for VirtualSocket state
- */
-#define ZT_CONNECT_RECHECK_DELAY 50
-
-/**
- * State check interval (in ms) for VirtualSocket state
- */
-#define ZT_API_CHECK_INTERVAL 50
-
-/**
- * Size of TCP TX buffer for VirtualSockets used in raw network stack drivers
- */
-#define ZT_TCP_TX_BUF_SZ 1024 * 1024 * 128
-
-/**
- * Size of TCP RX buffer for VirtualSockets used in raw network stack drivers
- */
-#define ZT_TCP_RX_BUF_SZ 1024 * 1024 * 128
-
-/**
- * Size of UDP TX buffer for VirtualSockets used in raw network stack drivers
- */
-#define ZT_UDP_TX_BUF_SZ ZT_MAX_MTU
-
-/**
- * Size of UDP RX buffer for VirtualSockets used in raw network stack drivers
- */
-#define ZT_UDP_RX_BUF_SZ ZT_MAX_MTU * 10
-
-/**
- * Send buffer size for the network stack
- * By default picoTCP sets them to 16834, this is good for embedded-scale
- * stuff but you might want to consider higher values for desktop and mobile
- * applications.
- */
-#define ZT_STACK_TCP_SOCKET_TX_SZ ZT_TCP_TX_BUF_SZ
-
-/**
- * Receive buffer size for the network stack
- * By default picoTCP sets them to 16834, this is good for embedded-scale
- * stuff but you might want to consider higher values for desktop and mobile
- * applications.
- */
-#define ZT_STACK_TCP_SOCKET_RX_SZ ZT_TCP_RX_BUF_SZ
-
-/**
- * Maximum size we're allowed to read or write from a stack socket
- * This is put in place because picoTCP seems to fail at higher values.
- * If you use another stack you can probably bump this up a bit.
- */
-#define ZT_STACK_SOCKET_WR_MAX 4096
-
-/**
- * Maximum size of read operation from a network stack
- */
-#define ZT_STACK_SOCKET_RD_MAX 4096*4
-
-/**
- * Maximum length of libzt/ZeroTier home path (where keys, and config files are stored)
- */
-#define ZT_HOME_PATH_MAX_LEN 256
-
-/**
- * Length of human-readable MAC address string
- */
-#define ZT_MAC_ADDRSTRLEN 18
-
-/**
- * Everything is ok
- */
-#define ZT_ERR_OK 0
-
-/**
- * Value returned during an internal failure at the VirtualSocket/VirtualTap layer
- */
-#define ZT_ERR_GENERAL_FAILURE -88
-
-/**
- * Whether sockets created will have SO_LINGER set by default
- */
-#define ZT_SOCK_BEHAVIOR_LINGER	false
-	
-/**
- * Length of time that VirtualSockets should linger (in seconds)
- */
-#define ZT_SOCK_BEHAVIOR_LINGER_TIME 3
-
-/**
- * Maximum wait time for socket closure if data is still present in the write queue
- */
-#define ZT_SDK_CLTIME 60
-
-/**
- * Interval for performing background tasks (such as adding routes) on VirtualTap objects (in seconds)
- */
-#define ZT_HOUSEKEEPING_INTERVAL 1
-
-/****************************************************************************/
-/* Socket API Signatures                                                    */
-/****************************************************************************/
-
-#define ZT_SETSOCKOPT_SIG int fd, int level, int optname, const void *optval, socklen_t optlen
-#define ZT_GETSOCKOPT_SIG int fd, int level, int optname, void *optval, socklen_t *optlen
-#define ZT_SENDMSG_SIG int fd, const struct msghdr *msg, int flags
-#define ZT_SENDTO_SIG int fd, const void *buf, size_t len, int flags, const struct sockaddr *addr, socklen_t addrlen
-#define ZT_RECV_SIG int fd, void *buf, size_t len, int flags
-#define ZT_RECVFROM_SIG int fd, void *buf, size_t len, int flags, struct sockaddr *addr, socklen_t *addrlen
-#define ZT_RECVMSG_SIG int fd, struct msghdr *msg,int flags
-#define ZT_SEND_SIG int fd, const void *buf, size_t len, int flags
-#define ZT_READ_SIG int fd, void *buf, size_t len
-#define ZT_WRITE_SIG int fd, const void *buf, size_t len
-#define ZT_SHUTDOWN_SIG int fd, int how
-#define ZT_SOCKET_SIG int socket_family, int socket_type, int protocol
-#define ZT_CONNECT_SIG int fd, const struct sockaddr *addr, socklen_t addrlen
-#define ZT_BIND_SIG int fd, const struct sockaddr *addr, socklen_t addrlen
-#define ZT_LISTEN_SIG int fd, int backlog
-#define ZT_ACCEPT4_SIG int fd, struct sockaddr *addr, socklen_t *addrlen, int flags
-#define ZT_ACCEPT_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
-#define ZT_CLOSE_SIG int fd
-#define ZT_POLL_SIG struct pollfd *fds, nfds_t nfds, int timeout
-#define ZT_SELECT_SIG int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout
-#define ZT_GETSOCKNAME_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
-#define ZT_GETPEERNAME_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
-#define ZT_GETHOSTNAME_SIG char *name, size_t len
-#define ZT_SETHOSTNAME_SIG const char *name, size_t len
-#define ZT_FCNTL_SIG int fd, int cmd, int flags
-#define ZT_IOCTL_SIG int fd, unsigned long request, void *argp
-#define ZT_SYSCALL_SIG long number, ...
-
-#define LIBZT_SERVICE_NOT_STARTED_STR "service not started yet, call zts_startjoin()"
-
-#endif // _H

include/lwIP.h

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -33,15 +33,30 @@
 #ifndef ZT_LWIP_HPP
 #define ZT_LWIP_HPP
 
-#include "libztDefs.h"
+#include "Debug.hpp"
 #include "lwip/err.h"
 
 namespace ZeroTier {
   class MAC;
   class Mutex;
+  class VirtualTap;
   struct InetAddress;
 }
 
+/**
+ * @brief Increase the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when we know the stack won't be used by any virtual taps
+ */
+void lwip_hibernate_driver();
+
+/**
+ * @brief Decrease the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when at least one virtual tap is active
+ */
+void lwip_wake_driver();
+
 /**
  * @brief Initialize network stack semaphores, threads, and timers.
  *
@@ -50,6 +65,27 @@ namespace ZeroTier {
  */
 void lwip_driver_init();
 
+/**
+ * @brief Shutdown the stack as completely as possible (not officially supported by lwIP)
+ *
+ * @usage This is to be called after it is determined that no further network activity will take place.
+ * The tcpip thread will be stopped, all interfaces will be brought down and all resources will 
+ * be deallocated. A full application restart will be required to bring the stack back online.
+ * @return
+ */
+void lwip_driver_shutdown();
+
+/**
+ * @brief Bring all interfaces down belonging to the given virtual tap interface
+ *
+ * @usage This is to be called when the application desires to stop all traffic processing in the 
+ * stack. Unlike lwip_driver_shutdown(), the application can easily resume traffic processing 
+ * by re-adding a virtual tap (and associated lwip netifs)
+ * @return
+ */
+void lwip_driver_set_tap_interfaces_down(void *tapref);
+void lwip_driver_set_all_interfaces_down();
+
 /**
  * @brief Initialize and start the DNS client
  *
@@ -99,7 +135,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p);
  * @param len Length of Ethernet frame
  * @return
  */
-void lwip_eth_rx(VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
+void lwip_eth_rx(ZeroTier::VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
 	const void *data, unsigned int len);
 
 #endif // _H

include/lwipopts.h

@@ -39,13 +39,56 @@
 #ifndef __LWIPOPTS_H__
 #define __LWIPOPTS_H__
 
+/*
+ * Provides short-named socket macros. Keep disabled
+ */
+#define LWIP_COMPAT_SOCKETS 0
+
+/**
+ * Respond to broadcast pings (default is unicast only)
+ */
+#define LWIP_BROADCAST_PING 0
+
+/**
+ * Respond to multicast pings (default is unicast only)
+ */
+#define LWIP_MULTICAST_PING 0
+
+/**
+ * Enables the ability to forward IP packets across network
+ * interfaces. If you are going to run lwIP on a device with only one network
+ * interface, define this to 0.
+ */
+#define IP_FORWARD 0
+
+/**
+ * Number of active MAC-IP address pairs cached.
+ */
+#define ARP_TABLE_SIZE 10
+
+/**
+ * LWIP_FIONREAD_LINUXMODE==0 (default): ioctl/FIONREAD returns the amount of
+ * pending data in the network buffer. This is the way windows does it. It's
+ * the default for lwIP since it is smaller.
+ * LWIP_FIONREAD_LINUXMODE==1: ioctl/FIONREAD returns the size of the next
+ * pending datagram in bytes. This is the way linux does it. This code is only
+ * here for compatibility.
+ */
+#define LWIP_FIONREAD_LINUXMODE 1
+/**
+ * Enable SO_RCVBUF processing.
+ */
+#define LWIP_SO_RCVBUF 1
+/**
+ * Enable SO_LINGER processing.
+ */
+#define LWIP_SO_LINGER 0
 /*
  *  Provides its own errno
  */
-
 #if __ANDROID__
 #define LWIP_PROVIDE_ERRNO          0
-#define SOCKLEN_T_DEFINED
+//#define SOCKLEN_T_DEFINED
 #elif !defined(_MSC_VER)
 #define LWIP_PROVIDE_ERRNO          1
 #endif
@@ -53,25 +96,22 @@
 /*
  * Provides core locking machinery
  */
-#define LWIP_TCPIP_CORE_LOCKING     0
+#define LWIP_TCPIP_CORE_LOCKING     1
 
 /*
  * Provides a macro to spoof the names of the lwip socket functions
  */
 #define LWIP_POSIX_SOCKETS_IO_NAMES 0
 
-#define LWIP_NOASSERT               1
+/*
+ *
+ */
+#define LWIP_NOASSERT               0
 /*
  *
  */
 #define LWIP_TIMERS                 1
 
-/*
- *
- */
-//#define LWIP_COMPAT_MUTEX 1
-//#define LWIP_COMPAT_MUTEX_ALLOWED 1
-
 /*
  * Provides network/host byte transformation macros
  */
@@ -80,11 +120,6 @@
 #define LWIP_DONT_PROVIDE_BYTEORDER_FUNCTIONS 1
 #endif
 
-/*
- * Include user defined options first. Anything not defined in these files
- * will be set to standard values. Override anything you dont like!
- */
-#include "lwip/debug.h"
 
 #define LWIP_IPV6_AUTOCONFIG 1
 
@@ -134,7 +169,7 @@
 // API
 #define API_LIB_DEBUG                   LWIP_DBG_OFF
 #define API_MSG_DEBUG                   LWIP_DBG_OFF
-#define SOCKETS_DEBUG                   LWIP_DBG_OFF
+#define SOCKETS_DEBUG                   LWIP_DBG_ON
 // other
 #define ICMP_DEBUG                      LWIP_DBG_OFF
 #define IGMP_DEBUG                      LWIP_DBG_OFF
@@ -177,9 +212,12 @@
 #define LWIP_CHKSUM_ALGORITHM 2
 
 #undef TCP_MSS
-#define TCP_MSS 1460
+
+#define MTU 1500
+#define TCP_MSS MTU - 40
 
 #define LWIP_NETIF_API 1
+#define LWIP_DEBUG_TIMERNAMES 1
 /*
 The TCP window size can be adjusted by changing the define TCP_WND. However,
 do keep in mind that this should be at least twice the size of TCP_MSS (thus
@@ -194,7 +232,6 @@ remote peer.
 
 #define TCP_WND 0xffff // max = 0xffff, min = TCP_MSS*2
 
-#define LWIP_NOASSERT 1
 #define TCP_LISTEN_BACKLOG   0
 
 /*------------------------------------------------------------------------------
@@ -252,12 +289,12 @@ happening sooner than they should.
 -------------------------------- Memory options --------------------------------
 ------------------------------------------------------------------------------*/
 
-//#define MEM_USE_POOLS 1
+//#define MEM_USE_POOLS 0
 //#define MEMP_USE_CUSTOM_POOLS 1
 
 /* Misc */
-#define MEM_LIBC_MALLOC 1
+#define MEM_LIBC_MALLOC 0
-#define MEMP_MEM_MALLOC 1
+#define MEMP_MEM_MALLOC 1 /* if set to 1, all dynamically allocated memory will come from MEM_SIZE */
 
 /**
  * MEM_ALIGNMENT: should be set to the alignment of the CPU
@@ -270,7 +307,7 @@ happening sooner than they should.
  * MEM_SIZE: the size of the heap memory. If the application will send
  * a lot of data that needs to be copied, this should be set high.
  */
-#define MEM_SIZE                        1024 * 1024 * 64
+#define MEM_SIZE                        1024 * 1024
 #define TCP_SND_BUF                     1024 * 63
 //#define TCP_OVERSIZE                    TCP_MSS
 
@@ -368,7 +405,7 @@ happening sooner than they should.
  * MEMP_NUM_NETCONN: the number of struct netconns.
  * (only needed if you use the sequential API, like api_lib.c)
  */
-#define MEMP_NUM_NETCONN                32
+#define MEMP_NUM_NETCONN                256
 
 /**
  * MEMP_NUM_TCPIP_MSG_API: the number of struct tcpip_msg, which are used
@@ -387,7 +424,7 @@ happening sooner than they should.
 /**
  * PBUF_POOL_SIZE: the number of buffers in the pbuf pool.
  */
-#define PBUF_POOL_SIZE                  4092 /* was 32 */
+#define PBUF_POOL_SIZE                  128 /* was 32 */
 
 
 /*------------------------------------------------------------------------------
@@ -595,6 +632,7 @@ happening sooner than they should.
  * LWIP_STATS==1: Enable statistics collection in lwip_stats.
  */
 #define LWIP_STATS                      1
+//#define LWIP_STATS_DISPLAY				1
 
 /*------------------------------------------------------------------------------
 --------------------------------- PPP Options ----------------------------------
@@ -605,4 +643,11 @@ happening sooner than they should.
  */
 #define PPP_SUPPORT                     0
 
+/*
+ * Include user defined options first. Anything not defined in these files
+ * will be set to standard values. Override anything you dont like!
+ */
+#include "lwip/debug.h"
+
 #endif /* __LWIPOPTS_H__ */
+

src/RingBuffer.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *

src/ServiceControls.cpp

@@ -0,0 +1,841 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+/**
+ * @file
+ *
+ * ZeroTier service controls
+ */
+
+
+
+#include "OneService.hpp"
+#include "Node.hpp"
+
+namespace ZeroTier
+{
+	std::vector<void*> vtaps;
+	Mutex _vtaps_lock;
+}
+
+#include "Constants.hpp"
+#include "VirtualTapManager.hpp"
+#include "lwIP.h"
+#include "OSUtils.hpp"
+#include "ServiceControls.hpp"
+
+//#define SDK_JNI 1
+#ifdef SDK_JNI
+#include <jni.h>
+#endif
+
+ZeroTier::Mutex _service_lock;
+ZeroTier::Mutex _startup_lock;
+
+static ZeroTier::OneService *zt1Service;
+std::string homeDir;
+int servicePort = ZTS_DEFAULT_PORT;
+bool _freeHasBeenCalled = false;
+bool _serviceIsShuttingDown = false;
+bool _startupError = false;
+
+#if defined(_WIN32)
+WSADATA wsaData;
+#include <Windows.h>
+#endif
+
+pthread_t service_thread;
+
+using namespace ZeroTier;
+
+//////////////////////////////////////////////////////////////////////////////
+// Internal ZeroTier Service Controls (user application shall not use these)//
+//////////////////////////////////////////////////////////////////////////////
+
+void api_sleep(int interval_ms)
+{
+#if defined(_WIN32)
+	Sleep(interval_ms);
+#else
+	struct timespec sleepValue = {0};
+	sleepValue.tv_nsec = interval_ms * 500000;
+	nanosleep(&sleepValue, NULL);
+#endif
+}
+
+int _zts_node_online()
+{
+	return zt1Service && zt1Service->getNode() && zt1Service->getNode()->online();
+}
+
+int _zts_can_perform_service_operation()
+{
+	return zt1Service && zt1Service->isRunning() && zt1Service->getNode() && zt1Service->getNode()->online() && !_serviceIsShuttingDown;
+}
+
+void _hibernate_if_needed()
+{
+	if (VirtualTapManager::get_vtaps_size()) {
+		lwip_wake_driver();
+	} else {
+		lwip_hibernate_driver();
+	}
+}
+#ifdef SDK_JNI
+#endif
+
+// Starts a ZeroTier service in the background
+#if defined(_WIN32)
+DWORD WINAPI _zts_start_service(LPVOID thread_id)
+#else
+void *_zts_start_service(void *thread_id)
+#endif
+{	
+	void *retval;
+	//DEBUG_INFO("identities are stored in path (%s)", homeDir.c_str());
+	zt1Service = (OneService *)0;
+
+	if (!homeDir.length()) {
+		DEBUG_ERROR("homeDir is empty, could not construct path");
+		_startupError = true;
+		retval = NULL;
+	} if (zt1Service) {
+		DEBUG_INFO("service already started, doing nothing");
+		retval = NULL;
+	}
+
+	try {
+		std::vector<std::string> hpsp(OSUtils::split(homeDir.c_str(), ZT_PATH_SEPARATOR_S,"",""));
+		std::string ptmp;
+		if (homeDir[0] == ZT_PATH_SEPARATOR) {
+			ptmp.push_back(ZT_PATH_SEPARATOR);
+		}
+		for (std::vector<std::string>::iterator pi(hpsp.begin());pi!=hpsp.end();++pi) {
+			if (ptmp.length() > 0) {
+				ptmp.push_back(ZT_PATH_SEPARATOR);
+			}
+			ptmp.append(*pi);
+			if ((*pi != ".")&&(*pi != "..")) {
+				if (OSUtils::mkdir(ptmp) == false) {
+					DEBUG_ERROR("home path does not exist, and could not create");
+					_startupError = true;
+					retval = NULL;
+					perror("error\n");
+				}
+			}
+		}
+		if (!_startupError) {
+			for(;;) {
+				_service_lock.lock();
+				zt1Service = OneService::newInstance(homeDir.c_str(),servicePort);
+				_service_lock.unlock();
+				switch(zt1Service->run()) {
+					case OneService::ONE_STILL_RUNNING:
+					case OneService::ONE_NORMAL_TERMINATION:
+						break;
+					case OneService::ONE_UNRECOVERABLE_ERROR:
+						DEBUG_ERROR("fatal error: %s", zt1Service->fatalErrorMessage().c_str());
+						_startupError = true;
+						break;
+					case OneService::ONE_IDENTITY_COLLISION: {
+						_startupError = true;
+						delete zt1Service;
+						zt1Service = (OneService *)0;
+						std::string oldid;
+						OSUtils::readFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid);
+						if (oldid.length()) {
+							OSUtils::writeFile((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid);
+							OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
+							OSUtils::rm((homeDir + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
+						}
+					}	continue; // restart!
+				}
+				break; // terminate loop -- normally we don't keep restarting
+			}
+		}
+		
+		_serviceIsShuttingDown = true;
+		_service_lock.lock();
+		delete zt1Service;
+		zt1Service = (OneService *)0;
+		_service_lock.unlock();
+		_serviceIsShuttingDown = false;
+	} catch ( ... ) {
+		DEBUG_ERROR("unexpected exception starting ZeroTier instance");
+	}
+	pthread_exit(NULL);
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Service Controls                                                //
+//////////////////////////////////////////////////////////////////////////////
+
+zts_err_t zts_set_service_port(int portno)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	_service_lock.lock();
+	if (zt1Service) {
+		// Stop service before attempting to set a port
+		retval = ZTS_ERR_SERVICE;
+	}
+	else {
+		if (portno > -1 && portno < ZTS_MAX_PORT) {
+			// 0 is allowed, signals to ZT service to bind to a random port
+			servicePort = portno;
+			retval = ZTS_ERR_OK;
+		}
+	}
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_set_1service_1port(
+	JNIEnv *env, jobject thisObj, jint port)
+{
+	zts_set_service_port(port);
+}
+#endif
+
+int zts_get_service_port()
+{
+	return servicePort;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_get_1service_1port(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_get_service_port();
+}
+#endif
+/*
+int zts_get_address(const uint64_t nwid, struct sockaddr_storage *addr, 
+	const int address_family)
+{
+	int err = -1;
+	if (!zt1Service) {
+		return ZTS_ERR_SERVICE;
+	}
+	VirtualTap *tap = getTapByNWID(nwid);
+	if (!tap) {
+		return -1;
+	}
+	_vtaps_lock.lock();
+	socklen_t addrlen = address_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	for (size_t i=0; i<tap->_ips.size(); i++) {
+		if (address_family == AF_INET) {
+			if (tap->_ips[i].isV4()) {
+				memcpy(addr, &(tap->_ips[i]), addrlen);
+				addr->ss_family = AF_INET;
+				err = 0;
+				break;
+			}
+		}
+		if (address_family == AF_INET6) {
+			if (tap->_ips[i].isV6()) {
+				memcpy(addr, &(tap->_ips[i]), addrlen);
+				addr->ss_family = AF_INET6;
+				err = 0;
+				break;
+			}
+		}
+	}
+	_vtaps_lock.unlock();
+	return err; // nothing found
+}
+#ifdef SDK_JNI
+JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_get_1address(
+	JNIEnv *env, jobject thisObj, jlong nwid, jint address_family, jobject addr)
+{
+	struct sockaddr_storage ss;
+	int err = zts_get_address((uint64_t)nwid, &ss, address_family);
+	ss2zta(env, &ss, addr);
+	return err;
+}
+#endif
+
+int zts_has_address(const uint64_t nwid)
+{
+	struct sockaddr_storage ss;
+	memset(&ss, 0, sizeof(ss));
+	zts_get_address(nwid, &ss, AF_INET);
+	if (ss.ss_family == AF_INET) {
+		return true;
+	}
+	zts_get_address(nwid, &ss, AF_INET6);
+	if (ss.ss_family == AF_INET6) {
+		return true;
+	}
+	return false;
+}
+#ifdef SDK_JNI
+JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_has_1address(
+	JNIEnv *env, jobject thisObj, jlong nwid)
+{
+	return zts_has_address(nwid);
+}
+#endif
+*/
+
+/*
+void zts_get_6plane_addr(struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId)
+{
+	ZeroTier::InetAddress _6planeAddr = ZeroTier::InetAddress::makeIpv66plane(nwid,nodeId);
+	memcpy(addr, _6planeAddr.rawIpData(), sizeof(struct sockaddr_storage));
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_16plane_1addr(
+	JNIEnv *env, jobject thisObj, jlong nwid, jlong nodeId, jobject addr)
+{
+	struct sockaddr_storage ss;
+	zts_get_6plane_addr(&ss, nwid, nodeId);
+	ss2zta(env, &ss, addr);
+}
+#endif
+
+void zts_get_rfc4193_addr(struct sockaddr_storage *addr, const uint64_t nwid, const uint64_t nodeId)
+{
+	ZeroTier::InetAddress _rfc4193Addr = ZeroTier::InetAddress::makeIpv6rfc4193(nwid,nodeId);
+	memcpy(addr, _rfc4193Addr.rawIpData(), sizeof(struct sockaddr_storage));
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_get_1rfc4193_1addr(
+	JNIEnv *env, jobject thisObj, jlong nwid, jlong nodeId, jobject addr)
+{
+	struct sockaddr_storage ss;
+	zts_get_rfc4193_addr(&ss, nwid, nodeId);
+	ss2zta(env, &ss, addr);
+}
+#endif
+*/
+
+
+zts_err_t zts_join(const uint64_t nwid, int blocking)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	retval = VirtualTapManager::get_vtaps_size() >= ZTS_MAX_JOINED_NETWORKS ? ZTS_ERR_INVALID_OP : ZTS_ERR_OK;
+	if (retval == ZTS_ERR_OK) {
+		_service_lock.lock();
+		if (blocking) {
+			if (!zt1Service) {
+				retval = ZTS_ERR_SERVICE;
+			} else {
+				while (!_zts_node_online()) {
+					if (_serviceIsShuttingDown) {
+						retval = ZTS_ERR_SERVICE;
+						break;
+					}
+					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+				}
+			}
+		} else {
+			if (!zt1Service || !_zts_node_online()) {
+				retval = ZTS_ERR_SERVICE;
+			}
+		}
+		if (retval == ZTS_ERR_OK) {
+			if (nwid == 0) {
+				retval = ZTS_ERR_INVALID_ARG;
+			}
+			if (zt1Service) {
+				zt1Service->getNode()->join(nwid, NULL, NULL);
+			}
+			VirtualTapManager::update_service_references((void*)zt1Service);
+		}
+		_service_lock.unlock();
+		_hibernate_if_needed();
+	}
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_join(
+	JNIEnv *env, jobject thisObj, jlong nwid)
+{
+	return zts_join((uint64_t)nwid);
+}
+#endif
+
+zts_err_t zts_leave(const uint64_t nwid, int blocking)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	_service_lock.lock();
+	if (blocking) {
+		if (!zt1Service) {
+			retval = ZTS_ERR_SERVICE;
+		} else {
+			while (!_zts_node_online()) {
+				if (_serviceIsShuttingDown) {
+					retval = ZTS_ERR_SERVICE;
+					break;
+				}
+				api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+			}
+		}
+	} else {
+		if (!zt1Service || !_zts_node_online()) {
+			retval = ZTS_ERR_SERVICE;
+		}
+	}
+	if (retval == ZTS_ERR_OK) {
+		if (nwid == 0) {
+			retval = ZTS_ERR_INVALID_ARG;
+		}
+		if (zt1Service) {
+			zt1Service->getNode()->leave(nwid, NULL, NULL);
+		}
+	}
+	VirtualTapManager::remove_by_nwid(nwid);
+	_hibernate_if_needed();
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_leave(
+	JNIEnv *env, jobject thisObj, jlong nwid)
+{
+	return zts_leave((uint64_t)nwid);
+}
+#endif
+
+zts_err_t zts_leave_all(int blocking)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else {
+		struct zts_network_details nds[ZTS_MAX_JOINED_NETWORKS];
+		int numJoined = ZTS_MAX_JOINED_NETWORKS;
+		zts_get_all_network_details(nds, &numJoined);
+		for (int i=0; i<numJoined; i++) {
+			zts_leave(nds[i].nwid);
+		}
+	}
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_orbit(uint64_t moonWorldId, uint64_t moonSeed)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	void *tptr = NULL;
+	_service_lock.lock();
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else if (_zts_can_perform_service_operation()) {
+		zt1Service->getNode()->orbit(tptr, moonWorldId, moonSeed);
+	}
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_deorbit(uint64_t moonWorldId)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	void *tptr = NULL;
+	_service_lock.lock();
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else if (_zts_can_perform_service_operation()) {
+		zt1Service->getNode()->deorbit(tptr, moonWorldId);
+	}
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+int zts_core_running()
+{
+	_service_lock.lock();
+	int retval = zt1Service == NULL ? false : zt1Service->isRunning();
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_core_1running(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_core_running();
+}
+#endif
+
+int zts_ready()
+{
+	_service_lock.lock();
+	bool stackRunning = VirtualTapManager::get_vtaps_size() > 0 ? true : false;
+	_service_lock.unlock();
+	return zts_core_running() && stackRunning;
+}
+#ifdef SDK_JNI
+JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_ready(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_ready();
+}
+#endif
+
+zts_err_t zts_start(const char *path, int blocking = false)
+{
+	_startup_lock.lock();
+	zts_err_t retval = ZTS_ERR_OK;
+	if (zt1Service) {
+		// Service is already initialized
+		retval = ZTS_ERR_SERVICE;
+	}
+	if (_freeHasBeenCalled) {
+		// Stack (presumably lwIP) has been dismantled, an application restart is required now
+		retval = ZTS_ERR_INVALID_OP;
+	}
+	if (!path) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (retval == ZTS_ERR_OK) {
+		homeDir = path;
+#if defined(_WIN32)
+		// initialize WinSock. Used in Phy for loopback pipe
+		WSAStartup(MAKEWORD(2, 2), &wsaData);
+		HANDLE thr = CreateThread(NULL, 0, _zts_start_service, NULL, 0, NULL);
+#else
+		_startupError = false;
+		retval = pthread_create(&service_thread, NULL, _zts_start_service, NULL);
+		// Wait for confirmation that the ZT service has been initialized, 
+		// this wait condition is so brief and so rarely used that it should be
+		// acceptable even in a non-blocking context.
+		while(!zt1Service) {
+			if (_serviceIsShuttingDown || _startupError) {
+				// ZT service startup/binding might have failed for some reason
+				retval = ZTS_ERR_SERVICE;
+				break;
+			}
+			api_sleep(10);
+		}
+#endif
+		if (blocking && retval == ZTS_ERR_OK) {
+			// block to prevent service calls before we're ready
+			// waiting for zerotier service thread to start
+			while (zts_core_running() == false || zt1Service->getNode() == NULL) {
+				if (_serviceIsShuttingDown || _startupError) {
+					// ZT service startup/binding might have failed for some reason
+					retval = ZTS_ERR_SERVICE;
+					break;
+				}
+				api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+			}
+			if (retval == ZTS_ERR_OK) {
+				// waiting for node address assignment
+				while (zt1Service->getNode()->address() <= 0) {
+					if (_serviceIsShuttingDown || _startupError) {
+						retval = ZTS_ERR_SERVICE;
+						break;
+					}
+					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+				}
+			}
+			if (retval == ZTS_ERR_OK) {
+				// Waiting for node to come online. Ensure the node is authorized to join the network
+				while (true) {
+					_service_lock.lock();
+					if (_serviceIsShuttingDown || _startupError) {
+						retval = ZTS_ERR_SERVICE;
+						break;
+					}
+					if (zt1Service && zt1Service->getNode() && zt1Service->getNode()->online()) {
+						// Node is fully online
+						break;
+					}
+					api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+					_service_lock.unlock();
+				}
+				_service_lock.unlock();
+			}
+		}
+	}
+	_startup_lock.unlock();
+	// if (blocking && retval == ZTS_ERR_OK) { DEBUG_INFO("node=%llx online", (unsigned long long)zts_get_node_id());}
+	_hibernate_if_needed();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_start(
+	JNIEnv *env, jobject thisObj, jstring path, jboolean blocking)
+{
+	if (path) {
+		const char* utf_string = env->GetStringUTFChars(path, NULL);
+		zts_start(utf_string, blocking);
+		env->ReleaseStringUTFChars(path, utf_string);
+	}
+}
+#endif
+
+zts_err_t zts_startjoin(const char *path, const uint64_t nwid)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if ((retval = zts_start(path, true)) < 0) {
+		return retval;
+	}
+	while (true) {
+		try {
+			zts_join(nwid);
+			break;
+		}
+		catch( ... ) {
+			api_sleep(ZTS_WRAPPER_CHECK_INTERVAL);
+			retval = ZTS_ERR_SERVICE;
+		}
+	}
+	_hibernate_if_needed();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_startjoin(
+	JNIEnv *env, jobject thisObj, jstring path, jlong nwid)
+{
+	if (path && nwid) {
+		const char* utf_string = env->GetStringUTFChars(path, NULL);
+		zts_startjoin(utf_string, (uint64_t)nwid);
+		env->ReleaseStringUTFChars(path, utf_string);
+	}
+}
+#endif
+
+zts_err_t zts_stop(int blocking)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	_service_lock.lock();
+	bool didStop = false;
+	if (_zts_can_perform_service_operation()) {
+		didStop = true;
+		zt1Service->terminate();
+		VirtualTapManager::clear();
+	}
+	else {
+		// Nothing to do
+		retval = ZTS_ERR_SERVICE;
+	}
+#if defined(_WIN32)
+	WSACleanup();
+#endif
+	_service_lock.unlock();
+	if (blocking && retval == ZTS_ERR_OK && didStop) {
+		// Block until ZT service thread successfully exits
+		pthread_join(service_thread, NULL);
+	}
+	_hibernate_if_needed();
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_stop(
+	JNIEnv *env, jobject thisObj)
+{
+	zts_stop();
+}
+#endif
+
+zts_err_t zts_free()
+{
+	zts_err_t retval = 0;
+	_service_lock.lock();
+	if (_freeHasBeenCalled) {
+		retval = ZTS_ERR_INVALID_OP;
+		_service_lock.unlock();
+	} else {
+		_freeHasBeenCalled = true;
+		_service_lock.unlock();
+		retval = zts_stop();
+	}
+	// PENDING: add stack shutdown logic
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT void JNICALL Java_com_zerotier_libzt_ZeroTier_free(
+	JNIEnv *env, jobject thisObj)
+{
+	zts_free();
+}
+#endif
+
+uint64_t zts_get_node_id()
+{
+	uint64_t nodeId = 0;
+	_service_lock.lock();
+	if (_zts_can_perform_service_operation()) {
+		nodeId = zt1Service->getNode()->address();
+	}
+	_service_lock.unlock();
+	return nodeId;
+}
+#ifdef SDK_JNI
+JNIEXPORT jlong JNICALL Java_com_zerotier_libzt_ZeroTier_get_1node_1id(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_get_node_id();
+}
+#endif
+
+int zts_get_peer_count()
+{
+	unsigned int peerCount = 0;
+	_service_lock.lock();
+	if (_zts_can_perform_service_operation()) {
+		peerCount = zt1Service->getNode()->peers()->peerCount;
+	} else {
+		peerCount = ZTS_ERR_SERVICE;
+	}
+	_service_lock.unlock();
+	return peerCount;
+}
+#ifdef SDK_JNI
+JNIEXPORT jlong JNICALL Java_com_zerotier_libzt_ZeroTier_get_1peer_1count(
+	JNIEnv *env, jobject thisObj)
+{
+	return zts_get_peer_count();
+}
+#endif
+
+int zts_get_peers(struct zts_peer_details *pds, int *num)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!pds || !num) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (retval == ZTS_ERR_OK) {
+		_service_lock.lock();
+		if (_zts_can_perform_service_operation()) {
+			ZT_PeerList *pl = zt1Service->getNode()->peers();
+			if (pl) {
+				*num = pl->peerCount;
+				for(unsigned long i=0;i<pl->peerCount;++i) {
+					memcpy(&(pds[i]), &(pl->peers[i]), sizeof(struct zts_peer_details));
+				}
+			}
+		}
+		else {
+			retval = ZTS_ERR_SERVICE;
+		}
+		_service_lock.unlock();
+	}
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_get_num_joined_networks()
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else {
+		retval = VirtualTapManager::get_vtaps_size();
+	}
+	return retval;
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_get_1num_1joined_1networks(
+		JNIEnv *env, jobject thisObj)
+{
+	return zts_get_num_joined_networks();
+}
+#endif
+
+zts_err_t zts_get_network_details(uint64_t nwid, struct zts_network_details *nd)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!nd || nwid == 0) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	if (retval == ZTS_ERR_OK) {
+		VirtualTapManager::get_network_details(nwid, nd);
+	}
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_get_all_network_details(struct zts_network_details *nds, int *num)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (!nds || !num) {
+		retval = ZTS_ERR_INVALID_ARG;
+	}
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	if (retval == ZTS_ERR_OK) {
+		VirtualTapManager::get_all_network_details(nds, num);
+	}
+	return retval;	
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_enable_http_backplane_mgmt()
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	_service_lock.lock();
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else {
+		zt1Service->allowHttpBackplaneManagement = true;
+	}
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+zts_err_t zts_disable_http_backplane_mgmt()
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	_service_lock.lock();
+	if (!zt1Service || _freeHasBeenCalled || _serviceIsShuttingDown) {
+		retval = ZTS_ERR_SERVICE;
+	}
+	else {
+		zt1Service->allowHttpBackplaneManagement = false;
+	}
+	_service_lock.unlock();
+	return retval;
+}
+#ifdef SDK_JNI
+#endif
+
+#ifdef __cplusplus
+}
+#endif

src/VirtualTap.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -25,33 +25,25 @@
  */
 
 /**
-* @file
+ * @file
-*
+ *
-* Virtual Ethernet tap device
+ * Virtual Ethernet tap device
-*/
+ */
-
-// Used by raw stack driver implementation
-struct ip_addr_t;
-typedef unsigned short u16_t;
 
+#include "VirtualTap.hpp"
 #include "Phy.hpp"
-
+#include "Node.hpp"
-#include "VirtualTap.h"
+#include "OSUtils.hpp"
-#include "libzt.h"
-#include "libztDebug.h"
-#include "lwIP.h"
-#include "ZT1Service.h"
-#include "SysUtils.h"
 
 #include "Mutex.hpp"
-#include "InetAddress.hpp"
+#include "VirtualTapManager.hpp"
-#include "OneService.hpp"
+#include "lwIP.h"
 
 #ifdef _MSC_VER
 #include "Synchapi.h"
 #endif
 
-int VirtualTap::devno = 0;
+namespace ZeroTier {
 
 VirtualTap::VirtualTap(
 	const char *homePath,
@@ -75,17 +67,11 @@ VirtualTap::VirtualTap(
 		_unixListenSocket((PhySocket *)0),
 		_phy(this,false,true)
 {
-	vtaps.push_back((void*)this);
+	VirtualTapManager::add_tap(this);
-
-	// set virtual tap interface name (full)
 	memset(vtap_full_name, 0, sizeof(vtap_full_name));
-	ifindex = devno;
+	snprintf(vtap_full_name, sizeof(vtap_full_name), "libzt%llx", (unsigned long long)_nwid);
-	snprintf(vtap_full_name, sizeof(vtap_full_name), "libzt%d-%llx", devno++, (unsigned long long)_nwid);
 	_dev = vtap_full_name;
-	DEBUG_INFO("set VirtualTap interface name to: %s", _dev.c_str());
+	::pipe(_shutdownSignalPipe);
-	// set virtual tap interface name (abbreviated)
-	memset(vtap_abbr_name, 0, sizeof(vtap_abbr_name));
-	snprintf(vtap_abbr_name, sizeof(vtap_abbr_name), "libzt%d", devno);
 	lwip_driver_init();
 	// start virtual tap thread and stack I/O loops
 	_thread = Thread::start(this);
@@ -93,10 +79,13 @@ VirtualTap::VirtualTap(
 
 VirtualTap::~VirtualTap()
 {
+	lwip_driver_set_tap_interfaces_down(this);
 	_run = false;
+	::write(_shutdownSignalPipe[1],"\0",1);
 	_phy.whack();
 	Thread::join(_thread);
-	_phy.close(_unixListenSocket,false);
+	::close(_shutdownSignalPipe[0]);
+	::close(_shutdownSignalPipe[1]);
 }
 
 void VirtualTap::setEnabled(bool en)
@@ -117,7 +106,7 @@ void VirtualTap::registerIpWithStack(const InetAddress &ip)
 bool VirtualTap::addIp(const InetAddress &ip)
 {
 	char ipbuf[INET6_ADDRSTRLEN];
-	DEBUG_EXTRA("addr=%s, nwid=%llx", ip.toString(ipbuf), (unsigned long long)_nwid);
+	// DEBUG_INFO("addr=%s, nwid=%llx", ip.toString(ipbuf), (unsigned long long)_nwid);
 	Mutex::Lock _l(_ips_m);
 	registerIpWithStack(ip);
 	if (std::find(_ips.begin(),_ips.end(),ip) == _ips.end()) {
@@ -164,9 +153,9 @@ std::string VirtualTap::deviceName() const
 std::string VirtualTap::nodeId() const
 {
 	if (zt1ServiceRef) {
-		char id[ZTO_ID_LEN];
+		char id[ZTS_ID_LEN];
 		memset(id, 0, sizeof(id));
-		sprintf(id, "%llx", (unsigned long long)((ZeroTier::OneService *)zt1ServiceRef)->getNode()->address());
+		sprintf(id, "%llx", (unsigned long long)((OneService *)zt1ServiceRef)->getNode()->address());
 		return std::string(id);
 	}
 	else {
@@ -176,7 +165,7 @@ std::string VirtualTap::nodeId() const
 
 void VirtualTap::setFriendlyName(const char *friendlyName)
 {
-	DEBUG_EXTRA("%s", friendlyName);
+	DEBUG_INFO("%s", friendlyName);
 }
 
 void VirtualTap::scanMulticastGroups(std::vector<MulticastGroup> &added,
@@ -211,33 +200,57 @@ void VirtualTap::setMtu(unsigned int mtu)
 void VirtualTap::threadMain()
 	throw()
 {
+	fd_set readfds,nullfds;
+	struct timeval tv;
+	tv.tv_sec = 0;
+	tv.tv_usec = 0;
+	FD_ZERO(&readfds);
+	FD_ZERO(&nullfds);
+	int nfds = (int)std::max(_shutdownSignalPipe[0],0) + 1;
 	while (true) {
+		FD_SET(_shutdownSignalPipe[0],&readfds);
+		select(nfds,&readfds,&nullfds,&nullfds,&tv);
+		if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) { // writes to shutdown pipe terminate thread
+			break;
+		}
 #ifdef _MSC_VER
-		Sleep(ZT_PHY_POLL_INTERVAL);
+		Sleep(ZTS_PHY_POLL_INTERVAL);
 		_phy.poll(0);
 #else
-		_phy.poll(ZT_PHY_POLL_INTERVAL);
+		_phy.poll(ZTS_PHY_POLL_INTERVAL);
 #endif
-		Housekeeping();
+
+		uint64_t current_ts = OSUtils::now();
+		if (current_ts > last_housekeeping_ts + ZTS_HOUSEKEEPING_INTERVAL) {
+			Housekeeping();
+			last_housekeeping_ts = OSUtils::now();
+		}
 	}
 }
 
 void VirtualTap::Housekeeping()
 {
 	Mutex::Lock _l(_tcpconns_m);
-	uint64_t current_ts = time_now();
+	OneService *service = ((OneService *)zt1ServiceRef);
-	if (current_ts > last_housekeeping_ts + ZT_HOUSEKEEPING_INTERVAL) {
+	if (!service) {
-		// update managed routes (add/del from network stacks)
+		return;
-		ZeroTier::OneService *service = ((ZeroTier::OneService *)zt1ServiceRef);
+	}
-		if (service) {
+	nd.num_routes = ZTS_MAX_NETWORK_ROUTES;
-			std::unique_ptr<std::vector<ZT_VirtualNetworkRoute>> managed_routes(service->getRoutes(this->_nwid));
+	service->getRoutes(this->_nwid, (ZT_VirtualNetworkRoute*)&(nd.routes)[0], &(nd.num_routes));
-			ZeroTier::InetAddress target_addr;
+
-			ZeroTier::InetAddress via_addr;
+/*
-			ZeroTier::InetAddress null_addr;
+
-			ZeroTier::InetAddress nm;
+			
+			InetAddress target_addr;
+			InetAddress via_addr;
+			InetAddress null_addr;
+			InetAddress nm;
 			null_addr.fromString("");
 			bool found;
 			char ipbuf[INET6_ADDRSTRLEN], ipbuf2[INET6_ADDRSTRLEN], ipbuf3[INET6_ADDRSTRLEN];
+*/
+
+
 			// TODO: Rework this when we have time
 			// check if pushed route exists in tap (add)
 			/*
@@ -258,7 +271,7 @@ void VirtualTap::Housekeeping()
 				if (found == false) {
 					if (via_addr.ipsEqual(null_addr) == false) {
 						DEBUG_INFO("adding route <target=%s, nm=%s, via=%s>", target_addr.toString(ipbuf), nm.toString(ipbuf2), via_addr.toString(ipbuf3));
-						routes.push_back(std::pair<ZeroTier::InetAddress,ZeroTier::InetAddress>(target_addr, nm));
+						routes.push_back(std::pair<InetAddress,InetAddress>(target_addr, nm));
 						routeAdd(target_addr, nm, via_addr);
 					}
 				}
@@ -281,15 +294,13 @@ void VirtualTap::Housekeeping()
 				}
 			}
 			*/
-		}
+		//}
 		// TODO: Clean up VirtualSocket objects
-		last_housekeeping_ts = time_now();
-	}
 }
 
-/****************************************************************************/
+//////////////////////////////////////////////////////////////////////////////
-/* Not used in this implementation                                          */
+// Not used in this implementation                                          //
-/****************************************************************************/
+//////////////////////////////////////////////////////////////////////////////
 
 void VirtualTap::phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *local_address,
 	const struct sockaddr *from,void *data,unsigned long len) {}
@@ -300,3 +311,4 @@ void VirtualTap::phyOnTcpClose(PhySocket *sock,void **uptr) {}
 void VirtualTap::phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 void VirtualTap::phyOnTcpWritable(PhySocket *sock,void **uptr) {}
 
+} // namespace ZeroTier

src/libzt.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -27,99 +27,328 @@
 /**
  * @file
  *
- * Application-facing, socket-like API
+ * ZeroTier Socket API
  */
 
-#include "libztDefs.h"
+#include <string.h>
 
 #include "lwip/sockets.h"
-#include "lwip/ip_addr.h"
+#include "Defs.hpp"
-#include "lwip/netdb.h"
+#include "libzt.h"
+#include "Debug.hpp"
 
-#include <string.h>
+#ifdef SDK_JNI
+	#include <jni.h>
+#ifndef _MSC_VER
+	//#include <sys/socket.h>
+	//#include <sys/types.h>
+	//#include <sys/select.h>
+	//#include <sys/ioctl.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+	void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+	void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+	void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set);
+	void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set);
+#ifdef __cplusplus
+}
+#endif
+#endif
+
+//////////////////////////////////////////////////////////////////////////////
+// Socket API                                                               //
+//////////////////////////////////////////////////////////////////////////////
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int platform_adjusted_socket_family(int family);
+// lwIP prototypes copied from lwip/src/include/sockets.h
-void fix_addr_socket_family(struct sockaddr *addr);
+// Don't call these directly, call zts_* functions instead
-bool zts_ready();
+int lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
+int lwip_bind(int s, const struct sockaddr *name, socklen_t namelen);
+int lwip_shutdown(int s, int how);
+int lwip_getpeername (int s, struct sockaddr *name, socklen_t *namelen);
+int lwip_getsockname (int s, struct sockaddr *name, socklen_t *namelen);
+int lwip_getsockopt (int s, int level, int optname, void *optval, socklen_t *optlen);
+int lwip_setsockopt (int s, int level, int optname, const void *optval, socklen_t optlen);
+int lwip_close(int s);
+int lwip_connect(int s, const struct sockaddr *name, socklen_t namelen);
+int lwip_listen(int s, int backlog);
+int lwip_recv(int s, void *mem, size_t len, int flags);
+int lwip_read(int s, void *mem, size_t len);
+int lwip_recvfrom(int s, void *mem, size_t len, int flags,
+      struct sockaddr *from, socklen_t *fromlen);
+int lwip_send(int s, const void *dataptr, size_t size, int flags);
+int lwip_sendmsg(int s, const struct msghdr *message, int flags);
+int lwip_sendto(int s, const void *dataptr, size_t size, int flags,
+    const struct sockaddr *to, socklen_t tolen);
+int lwip_socket(int domain, int type, int protocol);
+int lwip_write(int s, const void *dataptr, size_t size);
+int lwip_writev(int s, const struct iovec *iov, int iovcnt);
+int lwip_select(int maxfdp1, fd_set *readset, fd_set *writeset, fd_set *exceptset,
+                struct timeval *timeout);
+int lwip_ioctl(int s, long cmd, void *argp);
+int lwip_fcntl(int s, int cmd, int val);
+
+// Copied from lwip/src/include/sockets.h and renamed to prevent a name collision
+// with system definitions
+struct lwip_sockaddr {
+  u8_t        sa_len;
+  sa_family_t sa_family;
+  char        sa_data[14];
+};
+
+//////////////////////////////////////////////////////////////////////////////
+// ZeroTier Socket API                                                      //
+//////////////////////////////////////////////////////////////////////////////
+
+int zts_ready();
 
 int zts_socket(int socket_family, int socket_type, int protocol)
 {
-	int socket_family_adj = platform_adjusted_socket_family(socket_family);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_socket(socket_family, socket_type, protocol);
-	return !zts_ready() ? -1 : lwip_socket(socket_family_adj, socket_type, protocol);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
+	JNIEnv *env, jobject thisObj, jint family, jint type, jint protocol)
+{
+	return zts_socket(family, type, protocol);
+}
+#endif
 
 int zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen)
 {
-	struct sockaddr_storage ss;
+	if (!addr) {
-	memcpy(&ss, addr, addrlen);
+		return ZTS_ERR_INVALID_ARG;
-	fix_addr_socket_family((struct sockaddr*)&ss);
+	}
-	return !zts_ready() ? -1 : lwip_connect(fd, (struct sockaddr*)&ss, addrlen);
+	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_connect(fd, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_connect(
+	JNIEnv *env, jobject thisObj, jint fd, jobject addr)
+{
+	struct sockaddr_storage ss;
+	zta2ss(env, &ss, addr);
+	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	return zts_connect(fd, (struct sockaddr *)&ss, addrlen);
+}
+#endif
 
 int zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen)
 {
-	struct sockaddr_storage ss;
+	if (!addr) {
-	memcpy(&ss, addr, addrlen);
+		return ZTS_ERR_INVALID_ARG;
-	fix_addr_socket_family((struct sockaddr*)&ss);
+	}
-	return !zts_ready() ? -1 : lwip_bind(fd, (struct sockaddr*)&ss, addrlen);
+	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_bind(fd, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_bind(
+	JNIEnv *env, jobject thisObj, jint fd, jobject addr)
+{
+	struct sockaddr_storage ss;
+	zta2ss(env, &ss, addr);
+	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	return zts_bind(fd, (struct sockaddr*)&ss, addrlen);
+}
+#endif
 
 int zts_listen(int fd, int backlog)
 {
-	return !zts_ready() ? -1 : lwip_listen(fd, backlog);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_listen(fd, backlog);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(
+	JNIEnv *env, jobject thisObj, jint fd, int backlog)
+{
+	return zts_listen(fd, backlog);
+}
+#endif
 
 int zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen)
 {
-	return !zts_ready() ? -1 : lwip_accept(fd, addr, addrlen);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_accept(fd, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept(
+	JNIEnv *env, jobject thisObj, jint fd, jobject addr, jint port)
+{
+	struct sockaddr_storage ss;
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	int err = zts_accept(fd, (struct sockaddr *)&ss, &addrlen);
+	ss2zta(env, &ss, addr);
+	return err;
+}
+#endif
 
 #if defined(__linux__)
 int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)
 {
-	return !zts_ready() ? -1 : -1; // lwip_accept4(fd, addr, addrlen, flags);
+	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP;
 }
 #endif
+#ifdef SDK_JNI
+#if defined(__linux__)
+ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept4(
+	JNIEnv *env, jobject thisObj, jint fd, jobject addr, jint port, jint flags)
+ {
+	struct sockaddr_storage ss;
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	int err = zts_accept4(fd, (struct sockaddr *)&ss, &addrlen, flags);
+	ss2zta(env, &ss, addr);
+	return err;
+}
+#endif
+#endif
 
 int zts_setsockopt(int fd, int level, int optname, const void *optval, socklen_t optlen)
 {
-	return !zts_ready() ? -1 : lwip_setsockopt(fd, level, optname, optval, optlen);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_setsockopt(fd, level, optname, optval, optlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
+	JNIEnv *env, jobject thisObj, jint fd, jint level, jint optname, jobject optval)
+{
+	jclass c = (*env).GetObjectClass(optval);
+	if (!c) {
+		return ZTS_ERR_INVALID_OP;
+	}
+	int optval_int = -1;
+
+	if (optname == ZTS_SO_BROADCAST
+		|| optname == ZTS_SO_KEEPALIVE
+		|| optname == ZTS_SO_REUSEADDR	
+		|| optname == ZTS_SO_REUSEPORT
+		|| optname == ZTS_TCP_NODELAY)
+	{
+		jfieldID fid = (*env).GetFieldID(c, "booleanValue", "B");
+		optval_int = (int)(*env).GetBooleanField(optval, fid);
+	}
+	if (optname == ZTS_IP_TTL	
+		|| optname == ZTS_IP_TOS
+		|| optname == ZTS_SO_LINGER
+		|| optname == ZTS_SO_RCVBUF
+		|| optname == ZTS_SO_SNDBUF)
+	{
+		jfieldID fid = (*env).GetFieldID(c, "integerValue", "I");
+		optval_int = (*env).GetIntField(optval, fid);
+	}
+	int optlen = sizeof(optval_int);
+	return zts_setsockopt(fd, level, optname, &optval_int, optlen);
+}
+#endif
 
 int zts_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
 {
-	return !zts_ready() ? -1 : lwip_getsockopt(fd, level, optname, optval, optlen);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getsockopt(fd, level, optname, optval, optlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
+	JNIEnv *env, jobject thisObj, jint fd, jint level, jint optname, jobject optval)
+{
+	jclass c = (*env).GetObjectClass(optval);
+	if (!c) {
+		return ZTS_ERR_INVALID_OP;
+	}
+	int optval_int;
+	int optlen; // Intentionally not used
+	int err = ZTS_ERR_OK;
+	err = zts_getsockopt(fd, level, optname, &optval_int, &optlen);
+	if (optname == ZTS_SO_BROADCAST
+		|| optname == ZTS_SO_KEEPALIVE
+		|| optname == ZTS_SO_REUSEADDR	
+		|| optname == ZTS_SO_REUSEPORT
+		|| optname == ZTS_TCP_NODELAY)
+	{
+		jfieldID fid = (*env).GetFieldID(c, "isBoolean", "B");
+		(*env).SetBooleanField(optval, fid, true);
+		fid = (*env).GetFieldID(c, "booleanValue", "B");
+		(*env).SetBooleanField(optval, fid, (bool)optval_int);
+	}
+	if (optname == ZTS_IP_TTL	
+		|| optname == ZTS_IP_TOS
+		|| optname == ZTS_SO_LINGER
+		|| optname == ZTS_SO_RCVBUF
+		|| optname == ZTS_SO_SNDBUF)
+	{
+		jfieldID fid = (*env).GetFieldID(c, "isInteger", "B");
+		(*env).SetBooleanField(optval, fid, true);
+		fid = (*env).GetFieldID(c, "integerValue", "I");
+		(*env).SetIntField(optval, fid, optval_int);
+	}
+	return err;
+}
+#endif
 
 int zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen)
 {
-	return !zts_ready() ? -1 : lwip_getsockname(fd, addr, addrlen);
+	if (!addr) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getsockname(fd, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_getsockname(JNIEnv *env, jobject thisObj,
+	jint fd, jobject addr)
+{
+	struct sockaddr_storage ss;
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	int err = zts_getsockname(fd, (struct sockaddr *)&ss, &addrlen);
+	ss2zta(env, &ss, addr);
+	return err;
+}
+#endif
 
 int zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen)
 {
-	return !zts_ready() ? -1 : lwip_getpeername(fd, addr, addrlen);
+	if (!addr) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_getpeername(fd, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getpeername(JNIEnv *env, jobject thisObj,
+	jint fd, jobject addr)
+{
+	struct sockaddr_storage ss;
+	int err = zts_getpeername(fd, (struct sockaddr *)&ss, (socklen_t *)sizeof(struct sockaddr_storage));
+	ss2zta(env, &ss, addr);
+	return err;
+}
+#endif
 
 int zts_gethostname(char *name, size_t len)
 {
-	return !zts_ready() ? -1 : -1; // TODO
+	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
 }
+#ifdef SDK_JNI
+#endif
 
 int zts_sethostname(const char *name, size_t len)
 {
-	return !zts_ready() ? -1 : -1; // TODO
+	return !zts_ready() ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
 }
+#ifdef SDK_JNI
+#endif
 
 struct hostent *zts_gethostbyname(const char *name)
 {
-	if (zts_ready() == false) {
+	return (struct hostent *)(!zts_ready() ? NULL : NULL);
-		return NULL;
-	}
 	// TODO: Test thread safety
 	/*
 	char buf[256];
@@ -138,19 +367,63 @@ struct hostent *zts_gethostbyname(const char *name)
 	
 	return lwip_gethostbyname(name);
 	*/
-	return NULL;
 }
+#ifdef SDK_JNI
+#endif
 
 int zts_close(int fd)
 {
-	return !zts_ready() ? -1 : lwip_close(fd);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_close(fd);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_close(
+	JNIEnv *env, jobject thisObj, jint fd)
+{
+	return zts_close(fd);
+}
+#endif
 
 int zts_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
 	struct timeval *timeout)
 {
-	return !zts_ready() ? -1 : lwip_select(nfds, readfds, writefds, exceptfds, timeout);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_select(nfds, readfds, writefds, exceptfds, timeout);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_select(JNIEnv *env, jobject thisObj,
+	jint nfds, jobject readfds, jobject writefds, jobject exceptfds, jint timeout_sec, jint timeout_usec)
+{
+	struct timeval _timeout;
+	_timeout.tv_sec  = timeout_sec;
+		_timeout.tv_usec = timeout_usec;
+		fd_set _readfds, _writefds, _exceptfds;   		
+		fd_set *r = NULL;
+		fd_set *w = NULL;
+		fd_set *e = NULL;
+		if (readfds) {
+			r = &_readfds;
+			ztfdset2fdset(env, nfds, readfds, &_readfds);
+		}
+		if (writefds) {
+			w = &_writefds;
+			ztfdset2fdset(env, nfds, writefds, &_writefds);
+		}
+		if (exceptfds) {
+			e = &_exceptfds;
+			ztfdset2fdset(env, nfds, exceptfds, &_exceptfds);
+		}
+	int err = zts_select(nfds, r, w, e, &_timeout);
+	if (readfds) {
+		fdset2ztfdset(env, nfds, &_readfds, readfds);
+	}
+	if (writefds) {
+			fdset2ztfdset(env, nfds, &_writefds, writefds);
+		}
+		if (exceptfds) {
+			fdset2ztfdset(env, nfds, &_exceptfds, exceptfds);
+		}
+	return err;
+}
+#endif
 
 int zts_fcntl(int fd, int cmd, int flags)
 {
@@ -166,131 +439,369 @@ int zts_fcntl(int fd, int cmd, int flags)
 		translated_flags = 1;
 	}
 #endif
-	return !zts_ready() ? -1 : lwip_fcntl(fd, cmd, translated_flags);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_fcntl(fd, cmd, translated_flags);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_fcntl(
+	JNIEnv *env, jobject thisObj, jint fd, jint cmd, jint flags)
+{
+	return zts_fcntl(fd, cmd, flags);
+}
+#endif
 
 int zts_ioctl(int fd, unsigned long request, void *argp)
 {
-	return !zts_ready() ? -1 : lwip_ioctl(fd, request, argp);
+	if (!argp) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_ioctl(fd, request, argp);
 }
+#ifdef SDK_JNI
+JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
+	JNIEnv *env, jobject thisObj, jint fd, jlong request, jobject argp)
+{
+	zts_err_t retval = ZTS_ERR_OK;
+	if (request == FIONREAD) {
+		DEBUG_ERROR("FIONREAD");
+		int bytesRemaining = 0;
+		retval = zts_ioctl(fd, request, &bytesRemaining);	
+		// set value in general object
+		jclass c = (*env).GetObjectClass(argp);
+		if (!c) {
+			return ZTS_ERR_INVALID_ARG;
+		}
+		jfieldID fid = (*env).GetFieldID(c, "integer", "I");
+		(*env).SetIntField(argp, fid, bytesRemaining);
+	}
+	if (request == FIONBIO) {
+		// TODO: double check
+		int meaninglessVariable = 0;
+		DEBUG_ERROR("FIONBIO");
+		retval = zts_ioctl(fd, request, &meaninglessVariable);		
+	}
+	return retval;
+}
+#endif
+
+ssize_t zts_send(int fd, const void *buf, size_t len, int flags)
+{
+	if (!buf || len <= 0) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_send(fd, buf, len, flags);
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_send(
+	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, int flags)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int w = zts_send(fd, data, env->GetArrayLength(buf), flags);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return w;
+}
+#endif
 
 ssize_t zts_sendto(int fd, const void *buf, size_t len, int flags, 
 	const struct sockaddr *addr, socklen_t addrlen)
 {
-	struct sockaddr_storage ss;
+	if (!addr || !buf || len <= 0) {
-	memcpy(&ss, addr, addrlen);
+		return ZTS_ERR_INVALID_ARG;
-	fix_addr_socket_family((struct sockaddr*)&ss);
+	}
-	return !zts_ready() ? -1 : lwip_sendto(fd, buf, len, flags, (struct sockaddr*)&ss, addrlen);
+	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_sendto(fd, buf, len, flags, addr, addrlen);
 }
-
+#ifdef SDK_JNI
-ssize_t zts_send(int fd, const void *buf, size_t len, int flags)
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(
+	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, jint flags, jobject addr)
 {
-	return !zts_ready() ? -1 : lwip_send(fd, buf, len, flags);
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	struct sockaddr_storage ss;
+	zta2ss(env, &ss, addr);
+	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	int w = zts_sendto(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, addrlen);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return w;
 }
+#endif
 
 ssize_t zts_sendmsg(int fd, const struct msghdr *msg, int flags)
 {
-	return !zts_ready() ? -1 : lwip_sendmsg(fd, msg, flags);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_sendmsg(fd, msg, flags);
 }
+#ifdef SDK_JNI
+#endif
 
 ssize_t zts_recv(int fd, void *buf, size_t len, int flags)
 {
-	return !zts_ready() ? -1 : lwip_recv(fd, buf, len, flags);
+	if (!buf) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_recv(fd, buf, len, flags);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recv(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf, jint flags)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int r = zts_recv(fd, data, env->GetArrayLength(buf), flags);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return r;
+}
+#endif
 
 ssize_t zts_recvfrom(int fd, void *buf, size_t len, int flags, 
 	struct sockaddr *addr, socklen_t *addrlen)
 {
-	return !zts_ready() ? -1 : lwip_recvfrom(fd, buf, len, flags, addr, addrlen);
+	if (!buf) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_recvfrom(fd, buf, len, flags, addr, addrlen);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recvfrom(
+	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, jint flags, jobject addr)
+{
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	struct sockaddr_storage ss;
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int r = zts_recvfrom(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, &addrlen);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	ss2zta(env, &ss, addr);
+	return r;
+}
+#endif
 
 ssize_t zts_recvmsg(int fd, struct msghdr *msg, int flags)
 {
-	return !zts_ready() ? -1 : -1; // Not currently implemented by stack
+	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // Not currently implemented by stack
 }
+#ifdef SDK_JNI
+#endif
 
 int zts_read(int fd, void *buf, size_t len)
 {
-	return !zts_ready() ? -1 : lwip_read(fd, buf, len);
+	if (!buf) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_read(fd, buf, len);
 }
+int zts_read_offset(int fd, void *buf, size_t offset, size_t len)
+{
+	if (!buf) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	char *cbuf = (char*)buf;
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_read(fd, &(cbuf[offset]), len);
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int r = zts_read(fd, data, env->GetArrayLength(buf));
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return r;
+}
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read_1offset(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf, jint offset, jint len)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int r = zts_read_offset(fd, data, offset, len);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return r;
+}
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read_1length(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf, jint len)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int r = zts_read(fd, data, len);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return r;
+}
+#endif
 
 int zts_write(int fd, const void *buf, size_t len)
 {
-	return !zts_ready() ? -1 : lwip_write(fd, buf, len);
+	if (!buf || len <= 0) {
+		return ZTS_ERR_INVALID_ARG;
+	}
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_write(fd, buf, len);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write__IB(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf)
+{
+	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
+	int w = zts_write(fd, data, env->GetArrayLength(buf));
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return w;
+}
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1offset(JNIEnv *env, jobject thisObj,
+	jint fd, jbyteArray buf, jint offset, jint len)
+{
+	void *data = env->GetPrimitiveArrayCritical(&(buf[offset]), NULL); // PENDING: check?
+	int w = zts_write(fd, data, len);
+	env->ReleasePrimitiveArrayCritical(buf, data, 0);
+	return w;
+}
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1byte(JNIEnv *env, jobject thisObj,
+	jint fd, jbyte buf)
+{
+	return zts_write(fd, &buf, 1);
+}
+#endif
 
 int zts_shutdown(int fd, int how)
 {
-	return !zts_ready() ? -1 : lwip_shutdown(fd, how);
+	return !zts_ready() ? ZTS_ERR_SERVICE : lwip_shutdown(fd, how);
 }
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_shutdown(
+	JNIEnv *env, jobject thisObj, int fd, int how)
+{
+	return zts_shutdown(fd, how);
+}
+#endif
 
 int zts_add_dns_nameserver(struct sockaddr *addr)
 {
-	return !zts_ready() ? -1 : -1; // TODO
+	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // TODO
 }
+#ifdef SDK_JNI
+#endif
 
 int zts_del_dns_nameserver(struct sockaddr *addr)
 {
-	return !zts_ready() ? -1 : -1; // TODO
+	return !zts_ready() ? ZTS_ERR_SERVICE : -1; // TODO
 }
+#ifdef SDK_JNI
+#endif
 
-/* The rationale for the following correctional methods is as follows:
+#ifdef SDK_JNI
-
+void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set)
-	Since we don't want the user of this library to worry about naming conflicts
-	with their native OS/platform's socket facilities we deliberately isolate what
-	is used by the user-space network stack and stack drivers from the user's
-	application. As a result of this, we must compensate for a few things on our
-	side. For instance, differing values for AF_INET6 on major operating systems, and
-	differing structure definitions for sockaddr.
-*/
-
-/* adjust socket_family value (when AF_INET6) for various platforms:
-	linux  : 10
-	macOS  : 30
-	windows: 23
-*/
-int platform_adjusted_socket_family(int family)
 {
-#if defined(__linux__)
+	jclass c = (*env).GetObjectClass(src_ztfd_set);
-	return family; // do nothing
+	if (!c) {
-#endif
+		return;
-#if defined(__APPLE__)
+	}
-	 return family == 30 ? AF_INET6 : family; // 10
+	FD_ZERO(dest_fd_set);
-#endif
+	jfieldID fid = env->GetFieldID(c, "fds_bits", "[B");
-#if defined(_WIN32)
+	jobject fdData = (*env).GetObjectField (src_ztfd_set, fid);
-	 if (family == 23) {
+	jbyteArray * arr = reinterpret_cast<jbyteArray*>(&fdData);
-		 return AF_INET6;
+	char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);	
-	 }
+	for (int i=0; i<nfds; i++) {
-	 if (family == 2) {
+		if (data[i] == 0x01)  {
-		 return AF_INET;
+			FD_SET(i, dest_fd_set);
-	 }
-	 return -1;
-#endif
-}
-
-void fix_addr_socket_family(struct sockaddr *addr)
-{
-#if defined(__linux__) || defined(_WIN32)
-	/* struct sockaddr on Linux and Windows don't contain an sa_len field
-	so we must adjust it here before feeding it into the stack. */
-	if (addr->sa_len == 2) {
-		if (addr->sa_family == 0) {
-			addr->sa_family = addr->sa_len;
-			addr->sa_len = 0;
 		}
 	}
-	if (addr->sa_len == 10 || addr->sa_len == 23 || addr->sa_len == 30) {
+	(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
-		if (addr->sa_family == 0) {
+	return;
-			addr->sa_family = addr->sa_len;
+}
-			addr->sa_len = 0;
+
+void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set)
+{
+	jclass c = (*env).GetObjectClass(dest_ztfd_set);
+	if (!c) {
+		return;
+	}
+	jfieldID fid = env->GetFieldID(c, "fds_bits", "[B");
+	jobject fdData = (*env).GetObjectField (dest_ztfd_set, fid);
+	jbyteArray * arr = reinterpret_cast<jbyteArray*>(&fdData);
+	char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);
+	for (int i=0; i<nfds; i++) {
+		if (FD_ISSET(i, src_fd_set)) {
+			data[i] = 0x01;
 		}
 	}
-	/* once we've moved the value to its anticipated location, convert it from
+	(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
-	its platform-specific value to one that the network stack can work with */
+	return;
-#endif
-	addr->sa_family = platform_adjusted_socket_family(addr->sa_family);
 }
 
+//////////////////////////////////////////////////////////////////////////////
+// Helpers (for moving data across the JNI barrier)                         //
+//////////////////////////////////////////////////////////////////////////////
+
+void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
+{
+	jclass c = (*env).GetObjectClass(addr);
+	if (!c) {
+		return;
+	}
+	if(ss->ss_family == AF_INET)
+	{
+		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
+		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
+		(*env).SetIntField(addr, fid, ntohs(in4->sin_port));
+		fid = (*env).GetFieldID(c,"_family", "I");
+		(*env).SetIntField(addr, fid, (in4->sin_family));
+		fid = env->GetFieldID(c, "_ip4", "[B");
+		jobject ipData = (*env).GetObjectField (addr, fid);
+		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);
+		char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);
+		memcpy(data, &(in4->sin_addr.s_addr), 4);
+		(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
+
+		return;
+	}
+	if(ss->ss_family == AF_INET6)
+	{
+		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
+		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
+		(*env).SetIntField(addr, fid, ntohs(in6->sin6_port));
+		fid = (*env).GetFieldID(c,"_family", "I");
+		(*env).SetIntField(addr, fid, (in6->sin6_family));
+		fid = env->GetFieldID(c, "_ip6", "[B");
+		jobject ipData = (*env).GetObjectField (addr, fid);
+		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);
+		char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);
+		memcpy(data, &(in6->sin6_addr.s6_addr), 16);
+		(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
+		return;
+	}
+}
+
+void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
+{
+	jclass c = (*env).GetObjectClass(addr);
+	if (!c) {
+		return;
+	}
+	jfieldID fid = (*env).GetFieldID(c, "_family", "I");
+	int family = (*env).GetIntField(addr, fid);
+	if (family == AF_INET)
+	{
+		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
+		fid = (*env).GetFieldID(c, "_port", "I");
+		in4->sin_port = htons((*env).GetIntField(addr, fid));
+		in4->sin_family = AF_INET;
+		fid = env->GetFieldID(c, "_ip4", "[B");
+		jobject ipData = (*env).GetObjectField (addr, fid);
+		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);
+		char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);
+		memcpy(&(in4->sin_addr.s_addr), data, 4);
+		(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
+		return;
+	}
+	if (family == AF_INET6)
+	{
+		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
+		jfieldID fid = (*env).GetFieldID(c, "_port", "I");
+		in6->sin6_port = htons((*env).GetIntField(addr, fid));
+		fid = (*env).GetFieldID(c,"_family", "I");
+		in6->sin6_family = AF_INET6;
+		fid = env->GetFieldID(c, "_ip6", "[B");
+		jobject ipData = (*env).GetObjectField (addr, fid);
+		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);
+		char *data = (char*)(*env).GetByteArrayElements(*arr, NULL);
+		memcpy(&(in6->sin6_addr.s6_addr), data, 16);
+		(*env).ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
+		return;
+	}
+}
+#endif // JNI
+
 #ifdef __cplusplus
 }
 #endif

src/lwIP.cpp

@@ -1,6 +1,6 @@
 /*
  * ZeroTier SDK - Network Virtualization Everywhere
- * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
  *
  * --
  *
@@ -27,27 +27,16 @@
 /**
  * @file
  *
- * lwIP network stack driver.
+ * lwIP network stack driver
- *
- * Calls made in this network stack driver may never block since all packet
- * processing (input and output) as well as timer processing (TCP mainly) is done
- * in a single execution context.
- *
  */
 
-#include "libztDefs.h"
+#include <vector>
 
-#include "VirtualTap.h"
+#include "MAC.hpp"
-class VirtualTap;
 
 #include "Mutex.hpp"
-#include "MAC.hpp"
+#include "Constants.hpp"
-#include "ZeroTierOne.h"
+#include "VirtualTap.hpp"
-
-#include "libzt.h"
-#include "SysUtils.h"
-#include "Utilities.h"
-#include "libztDebug.h"
 
 #include "netif/ethernet.h"
 #include "lwip/netif.h"
@@ -76,49 +65,44 @@ void ms_sleep(unsigned long ms)
 }
 #endif
 
-struct netif lwipInterfaces[10];
-int lwipInterfacesCount = 0;
-
 ZeroTier::Mutex _rx_input_lock_m;
 struct pbuf* lwip_frame_rxbuf[LWIP_MAX_GUARDED_RX_BUF_SZ];
 int lwip_frame_rxbuf_tot = 0;
 
-
+bool main_loop_exited = false;
 bool lwip_driver_initialized = false;
+bool has_already_been_initialized = false;
+int hibernationDelayMultiplier = 1;
+
 ZeroTier::Mutex driver_m;
 
-err_t tapif_init(struct netif *netif)
+std::vector<struct netif *> lwip_netifs;
+
+void lwip_hibernate_driver()
 {
-	// we do the actual initialization in elsewhere
+	hibernationDelayMultiplier = ZTS_HIBERNATION_MULTIPLIER;
-	return ERR_OK;
 }
 
-/*
+void lwip_wake_driver()
-static void tcp_timeout(void *data)
 {
-	DEBUG_EXTRA("");
+	hibernationDelayMultiplier = 1;
-	LWIP_UNUSED_ARG(data);
-#if TCP_DEBUG && LWIP_TCP
-	// tcp_debug_print_pcbs();
-#endif
-	sys_timeout(5000, tcp_timeout, NULL);
 }
-*/
 
-// callback for when the TCPIP thread has been successfully started
+// Callback for when the TCPIP thread has been successfully started
 static void tcpip_init_done(void *arg)
 {
 	sys_sem_t *sem;
 	sem = (sys_sem_t *)arg;
-	//netif_set_up(&lwipdev);
 	lwip_driver_initialized = true;
 	driver_m.unlock();
-	// sys_timeout(5000, tcp_timeout, NULL);
 	sys_sem_signal(sem);
 }
 
 void my_tcpip_callback(void *arg)
 {
+	if (main_loop_exited) {
+		return;
+	}
 	ZeroTier::Mutex::Lock _l(_rx_input_lock_m);
 	int loop_score = LWIP_FRAMES_HANDLED_PER_CORE_CALL; // max num of packets to read per polling call
 	// TODO: Optimize (use Ringbuffer)
@@ -130,42 +114,40 @@ void my_tcpip_callback(void *arg)
 		// Packet routing logic. Inputs packet into correct lwip netif interface depending on protocol type
 		struct ip_hdr *iphdr;
 		switch (((struct eth_hdr *)p->payload)->type)
-		{
+		{		
-#ifdef LIBZT_IPV6
 			case PP_HTONS(ETHTYPE_IPV6): {
 				iphdr = (struct ip_hdr *)((char *)p->payload + SIZEOF_ETH_HDR);
-				for (int i=0; i<lwipInterfacesCount; i++) {
+				for (size_t i=0; i<lwip_netifs.size(); i++) {
-					if (lwipInterfaces[i].output_ip6 && lwipInterfaces[i].output_ip6 == ethip6_output) {
+					if (lwip_netifs[i]->output_ip6 && 
-						if (lwipInterfaces[i].input(p, &lwipInterfaces[i]) != ERR_OK) {
+						lwip_netifs[i]->output_ip6 == ethip6_output) {
-							DEBUG_ERROR("packet input error (ipv6, p=%p, netif=%p)", p, &lwipInterfaces[i]);
+						if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
+							DEBUG_ERROR("packet input error (ipv6, p=%p, netif=%p)", p, &lwip_netifs[i]);
 							break;
 						}
 					}
 				}	
 			} break;
-#endif
-#ifdef LIBZT_IPV4
 			case PP_HTONS(ETHTYPE_IP): {
 				iphdr = (struct ip_hdr *)((char *)p->payload + SIZEOF_ETH_HDR);
-				for (int i=0; i<lwipInterfacesCount; i++) {
+				for (size_t i=0; i<lwip_netifs.size(); i++) {
-					if (lwipInterfaces[i].output &&  lwipInterfaces[i].output == etharp_output) {
+					if (lwip_netifs[i]->output && 
-						//if (lwipInterfaces[i].ip_addr.addr == iphdr->dest.addr || ip4_addr_isbroadcast_u32(iphdr->dest.addr, &lwipInterfaces[i])) {
+						lwip_netifs[i]->output == etharp_output) {
-						if (lwipInterfaces[i].ip_addr.u_addr.ip4.addr == iphdr->dest.addr || ip4_addr_isbroadcast_u32(iphdr->dest.addr, &lwipInterfaces[i])) {
+						if (lwip_netifs[i]->ip_addr.u_addr.ip4.addr == iphdr->dest.addr || 
-							if (lwipInterfaces[i].input(p, &lwipInterfaces[i]) != ERR_OK) {
+							ip4_addr_isbroadcast_u32(iphdr->dest.addr, lwip_netifs[i])) {
-								DEBUG_ERROR("packet input error (ipv4, p=%p, netif=%p)", p, &lwipInterfaces[i]);
+							if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
+								DEBUG_ERROR("packet input error (ipv4, p=%p, netif=%p)", p, &lwip_netifs[i]);
 								break;
 							}
 						}
 					}
 				}
 			} break;
-#endif
 			case PP_HTONS(ETHTYPE_ARP): {
-				for (int i=0; i<lwipInterfacesCount; i++) {
+				for (size_t i=0; i<lwip_netifs.size(); i++) {
-					if (lwipInterfaces[i].state) {
+					if (lwip_netifs[i]->state) {
 						pbuf_ref(p);
-						if (lwipInterfaces[i].input(p, &lwipInterfaces[i]) != ERR_OK) {
+						if (lwip_netifs[i]->input(p, lwip_netifs[i]) != ERR_OK) {
-							DEBUG_ERROR("packet input error (arp, p=%p, netif=%p)", p, &lwipInterfaces[i]);
+							DEBUG_ERROR("packet input error (arp, p=%p, netif=%p)", p, &lwip_netifs[i]);
 						}
 						break;
 					}
@@ -179,52 +161,106 @@ void my_tcpip_callback(void *arg)
 		loop_score--;
 	}
 	int count_final = lwip_frame_rxbuf_tot;
-	// move pbuf frame pointer address buffer by the number of frames successfully fed into the stack core
+	// Move pbuf frame pointer address buffer by the number of frames successfully fed into the stack core
 	if (count_initial - count_final > 0) {
 		memmove(lwip_frame_rxbuf, lwip_frame_rxbuf + count_final, count_initial - count_final);
 	}
 }
 
 // main thread which starts the initialization process
-static void main_thread(void *arg)
+static void main_lwip_driver_loop(void *arg)
 {
 	sys_sem_t sem;
 	LWIP_UNUSED_ARG(arg);
 	if (sys_sem_new(&sem, 0) != ERR_OK) {
 		DEBUG_ERROR("failed to create semaphore");
 	}
-	
 	tcpip_init(tcpip_init_done, &sem);
+	has_already_been_initialized = true;
 	sys_sem_wait(&sem);
-	DEBUG_EXTRA("stack thread init complete");
+	//DEBUG_INFO("stack thread init complete");
 
-	while(1) {
+	while(lwip_driver_initialized) {
 #if defined(_WIN32)
-		ms_sleep(LWIP_GUARDED_BUF_CHECK_INTERVAL);
+		ms_sleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*hibernationDelayMultiplier);
 #else
-		usleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*1000);
+		usleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*1000*hibernationDelayMultiplier);
 #endif
 		// Handle incoming packets from the core's thread context.
 		// If you feed frames into the core directly you will violate the core's thread model
 		tcpip_callback_with_block(my_tcpip_callback, NULL, 1);
 	}
-	sys_sem_wait(&sem); // block forever
+	main_loop_exited = true;
 }
 
-// initialize the lwIP stack
+// Initialize the lwIP stack
 void lwip_driver_init()
 {
-	driver_m.lock(); // unlocked from callback indicating completion of driver init
+	driver_m.lock(); // Unlocked from callback indicating completion of driver init
-	if (lwip_driver_initialized == true) {
+	if (has_already_been_initialized || lwip_driver_initialized) {
+		// Already initialized, skip
+		driver_m.unlock();
+		return;
+	} if (main_loop_exited) {
+		DEBUG_ERROR("stack has previously been shutdown an cannot be restarted.");
+		driver_m.unlock();
 		return;
 	}
 #if defined(_WIN32)
-	sys_init(); // required for win32 initializtion of critical sections
+	sys_init(); // Required for win32 init of critical sections
 #endif
-	sys_thread_new("main_thread", main_thread,
+	void *st = sys_thread_new("main_thread", main_lwip_driver_loop,
 		NULL, DEFAULT_THREAD_STACKSIZE, DEFAULT_THREAD_PRIO);
 }
 
+void lwip_driver_shutdown()
+{
+	if (main_loop_exited) {
+		return;
+	}
+	lwip_driver_initialized = false;
+	// Give the stack time to call the frame feed callback one last time before shutting everything down
+	int callbackInterval = LWIP_GUARDED_BUF_CHECK_INTERVAL*hibernationDelayMultiplier*1000;
+	usleep(callbackInterval*3);
+	while(!main_loop_exited) {
+		usleep(LWIP_GUARDED_BUF_CHECK_INTERVAL*1000);
+	}
+	if (tcpip_shutdown() == ERR_OK) {
+		sys_timeouts_free();
+	}
+}
+
+void lwip_driver_set_all_interfaces_down()
+{
+	for (size_t i=0; i<lwip_netifs.size(); i++) {
+		if (lwip_netifs[i]) {
+			netif_remove(lwip_netifs[i]);
+			netif_set_down(lwip_netifs[i]);
+			netif_set_link_down(lwip_netifs[i]);
+			delete lwip_netifs[i];
+		}
+	}
+	lwip_netifs.clear();
+}
+
+void lwip_driver_set_tap_interfaces_down(void *tapref)
+{
+	int sz_i = lwip_netifs.size();
+	std::vector<struct netif*>::iterator iter;
+	for (iter = lwip_netifs.begin(); iter != lwip_netifs.end(); ) {
+		struct netif *lp = *(iter);
+		if (lp->state == tapref) {
+			netif_remove(lp);
+			netif_set_down(lp);
+			netif_set_link_down(lp);
+			iter = lwip_netifs.erase(iter);
+		}
+		else {
+			++iter;
+		}
+	}
+}
+
 err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 {
 	struct pbuf *q;
@@ -232,7 +268,7 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 	char *bufptr;
 	int totalLength = 0;
 
-	VirtualTap *tap = (VirtualTap*)netif->state;
+	ZeroTier::VirtualTap *tap = (ZeroTier::VirtualTap*)netif->state;
 	bufptr = buf;
 	for (q = p; q != NULL; q = q->next) {
 		memcpy(bufptr, q->payload, q->len);
@@ -255,18 +291,20 @@ err_t lwip_eth_tx(struct netif *netif, struct pbuf *p)
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
-		char macBuf[ZT_MAC_ADDRSTRLEN], nodeBuf[ZTO_ID_LEN];
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[ZTS_ID_LEN];
-		mac2str(macBuf, ZT_MAC_ADDRSTRLEN, ethhdr->dest.addr);
+		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			ethhdr->dest.addr[0], ethhdr->dest.addr[1], ethhdr->dest.addr[2], 
+			ethhdr->dest.addr[3], ethhdr->dest.addr[4], ethhdr->dest.addr[5]);
 		ZeroTier::MAC mac;
 		mac.setTo(ethhdr->dest.addr, 6);
 		mac.toAddress(tap->_nwid).toString(nodeBuf);
-		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
+		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
-			ZeroTier::Utils::ntoh(ethhdr->type), beautify_eth_proto_nums(ZeroTier::Utils::ntoh(ethhdr->type)), flagbuf);
+			ZeroTier::Utils::ntoh(ethhdr->type), flagbuf);
 	}
 	return ERR_OK;
 }
 
-void lwip_eth_rx(VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
+void lwip_eth_rx(ZeroTier::VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, unsigned int etherType,
 	const void *data, unsigned int len)
 {
 	struct pbuf *p,*q;
@@ -278,13 +316,15 @@ void lwip_eth_rx(VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC
 	if (ZT_MSG_TRANSFER == true) {
 		char flagbuf[32];
 		memset(&flagbuf, 0, 32);
-		char macBuf[ZT_MAC_ADDRSTRLEN], nodeBuf[ZTO_ID_LEN];
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[ZTS_ID_LEN];
-		mac2str(macBuf, ZT_MAC_ADDRSTRLEN, ethhdr.dest.addr);
+		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			ethhdr.dest.addr[0], ethhdr.dest.addr[1], ethhdr.dest.addr[2], 
+			ethhdr.dest.addr[3], ethhdr.dest.addr[4], ethhdr.dest.addr[5]);
 		ZeroTier::MAC mac;
 		mac.setTo(ethhdr.src.addr, 6);
 		mac.toAddress(tap->_nwid).toString(nodeBuf);
-		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
+		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] proto=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
-			ZeroTier::Utils::ntoh(ethhdr.type), beautify_eth_proto_nums(ZeroTier::Utils::ntoh(ethhdr.type)), flagbuf);
+			ZeroTier::Utils::ntoh(ethhdr.type), flagbuf);
 	}
 
 	p = pbuf_alloc(PBUF_RAW, len+sizeof(struct eth_hdr), PBUF_POOL);
@@ -309,16 +349,16 @@ void lwip_eth_rx(VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC
 		DEBUG_ERROR("dropped packet: no pbufs available");
 		return;
 	}
-	if (lwipInterfacesCount <= 0) {
+	if (!lwip_netifs.size()) {
 		DEBUG_ERROR("there are no netifs set up to handle this packet. ignoring.");
 		return;
 	}
-	Mutex::Lock _l(_rx_input_lock_m);
+	ZeroTier::Mutex::Lock _l(_rx_input_lock_m);
 	if (lwip_frame_rxbuf_tot == LWIP_MAX_GUARDED_RX_BUF_SZ) {
 		DEBUG_ERROR("dropped packet -- guarded receive buffer full, adjust MAX_GUARDED_RX_BUF_SZ or LWIP_GUARDED_BUF_CHECK_INTERVAL");
 		return;
 	}
-	pbuf_ref(p); // Increment reference to allow user application to copy data from buffer -- Will be automatically deallocated by socket API
+	//pbuf_ref(p); // Increment reference to allow user application to copy data from buffer -- Will be automatically deallocated by socket API
 	lwip_frame_rxbuf[lwip_frame_rxbuf_tot] = p;
 	lwip_frame_rxbuf_tot += 1;
 }
@@ -339,6 +379,7 @@ void lwip_start_dhcp(void *netif)
 
 static void netif_status_callback(struct netif *netif)
 {
+	/*
 	DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n",
 		netif,
 		netif->name[0],
@@ -356,6 +397,7 @@ static void netif_status_callback(struct netif *netif)
 		netif->state,
 		netif->flags
 	);
+	*/
 }
 
 ZeroTier::MAC _mac;
@@ -364,7 +406,7 @@ static err_t netif_init_4(struct netif *netif)
 {
 	netif->hwaddr_len = 6;
 	netif->name[0]    = 'e';
-	netif->name[1]    = '0'+lwipInterfacesCount;
+	netif->name[1]    = '0'+lwip_netifs.size();
 	netif->linkoutput = lwip_eth_tx;
 	netif->output     = etharp_output;
 	netif->mtu        = ZT_MAX_MTU;
@@ -383,7 +425,7 @@ static err_t netif_init_6(struct netif *netif)
 {
 	netif->hwaddr_len = 6;
 	netif->name[0]    = 'e';
-	netif->name[1]    = '0'+(char)lwipInterfacesCount;
+	netif->name[1]    = '0'+(char)lwip_netifs.size();
 	netif->linkoutput = lwip_eth_tx;
 	netif->output     = etharp_output;
 	netif->output_ip6 = ethip6_output;
@@ -400,12 +442,15 @@ static err_t netif_init_6(struct netif *netif)
 
 void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier::InetAddress &ip)
 {
-	char ipbuf[INET6_ADDRSTRLEN], nmbuf[INET6_ADDRSTRLEN];
+	char ipbuf[INET6_ADDRSTRLEN];
-	char macbuf[ZT_MAC_ADDRSTRLEN];
+	char macbuf[ZTS_MAC_ADDRSTRLEN];
-	struct netif *lwipdev = &lwipInterfaces[lwipInterfacesCount];
+	struct netif *lwipdev = new struct netif;
+	lwip_netifs.push_back(lwipdev);
+
 	_mac = mac;
 
 	if (ip.isV4()) {
+		char nmbuf[INET6_ADDRSTRLEN];
 		static ip4_addr_t ipaddr, netmask, gw;
 		IP4_ADDR(&gw,127,0,0,1);
 		ipaddr.addr = *((u32_t *)ip.rawIpData());
@@ -413,8 +458,11 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 		netif_set_status_callback(lwipdev, netif_status_callback);
 		netif_add(lwipdev, &ipaddr, &netmask, &gw, NULL, netif_init_4, tcpip_input);
 		lwipdev->state = tapref;
-		mac2str(macbuf, ZT_MAC_ADDRSTRLEN, lwipdev->hwaddr);
+		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-		DEBUG_INFO("initialized netif as [mac=%s, addr=%s, nm=%s]", macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf));
+			lwipdev->hwaddr[0], lwipdev->hwaddr[1], lwipdev->hwaddr[2], 
+			lwipdev->hwaddr[3], lwipdev->hwaddr[4], lwipdev->hwaddr[5]);
+		DEBUG_INFO("initialized netif as [mac=%s, addr=%s, nm=%s]", 
+			macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf));
 	}
 	if (ip.isV6())
 	{
@@ -431,8 +479,10 @@ void lwip_init_interface(void *tapref, const ZeroTier::MAC &mac, const ZeroTier:
 		netif_set_default(lwipdev);
 		netif_set_up(lwipdev);
 		netif_set_link_up(lwipdev);
-		mac2str(macbuf, ZT_MAC_ADDRSTRLEN, lwipdev->hwaddr);
+		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-		DEBUG_INFO("initialized netif as [mac=%s, addr=%s]", macbuf, ip.toString(ipbuf));
+			lwipdev->hwaddr[0], lwipdev->hwaddr[1], lwipdev->hwaddr[2],
+			lwipdev->hwaddr[3], lwipdev->hwaddr[4], lwipdev->hwaddr[5]);
+		DEBUG_INFO("initialized netif as [mac=%s, addr=%s]", 
+			macbuf, ip.toString(ipbuf));
 	}
-	lwipInterfacesCount++;
 }