Add portability and consistency fixes for C API, remove cruft, slight internal restructuring

CMakeLists.txt

@@ -29,10 +29,6 @@ endif ()
 if (${CMAKE_GENERATOR} STREQUAL "Xcode")
 	set (IN_XCODE TRUE)
 	set(XCODE_EMIT_EFFECTIVE_PLATFORM_NAME ON)
-	#set_target_properties (${STATIC_LIB_NAME}
-	#	PROPERTIES XCODE_ATTRIBUTE_MY_BUILD_ONLY_ACTIVE_ARCH YES)
-	#set_target_properties (${STATIC_LIB_NAME}
-	#	PROPERTIES XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH "${MY_BUILD_ONLY_ACTIVE_ARCH})
 endif ()
 if (BUILDING_WIN32 OR BUILDING_WIN64 OR MSVC)
 	set (BUILDING_WIN TRUE)
@@ -98,8 +94,6 @@ set (SILENCE "${SILENCE} -Wno-sign-compare")
 set (SILENCE "${SILENCE} -Wno-unused-variable")
 set (SILENCE "${SILENCE} -Wno-missing-field-initializers")
 set (SILENCE "${SILENCE} -Wno-unused-parameter")
-#set (SILENCE "${SILENCE} -Wno-nullability-completeness")
-#set (SILENCE "${SILENCE} -Wno-expansion-to-defined")
 
 set (ZT_FLAGS "${ZT_FLAGS} -DZT_USE_MINIUPNPC=1")
 set (ZT_FLAGS "${ZT_FLAGS} -DZT_SOFTWARE_UPDATE_DEFAULT=0")
@@ -109,8 +103,8 @@ set (ZT_FLAGS "${ZT_FLAGS} -DZT_SDK=1")
 if (DEBUG_BUILD)
 	set (LWIP_FLAGS "${LWIP_FLAGS} -DLWIP_DBG_TYPES_ON=128")
 	set (LWIP_FLAGS "${LWIP_FLAGS} -DSOCKETS_DEBUG=128")
-	set (LWIP_FLAGS "${LWIP_FLAGS} -DLWIP_STATS_LARGE=1")
+	#set (LWIP_FLAGS "${LWIP_FLAGS} -DLWIP_STATS_LARGE=1")
-	set (LWIP_FLAGS "${LWIP_FLAGS} -DLWIP_STATS=1")
+	#set (LWIP_FLAGS "${LWIP_FLAGS} -DLWIP_STATS=1")
 	set (LWIP_FLAGS "${LWIP_FLAGS} -DLINK_STATS=1")
 	set (LWIP_FLAGS "${LWIP_FLAGS} -DETHARP_STATS=1")
 	set (LWIP_FLAGS "${LWIP_FLAGS} -DIPFRAG_STATS=1")
@@ -250,31 +244,6 @@ set (LWIP_SRC_DIR "${PROJ_DIR}/ext/lwip/src")
 set (ZTO_SRC_DIR "${PROJ_DIR}/ext/ZeroTierOne")
 set (LIBZT_SRC_DIR "${PROJ_DIR}/src")
 
-if (SWIG_CSHARP)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/csharp/*.cxx)
-endif ()
-if (SWIG_PYTHON)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/python/*.cxx)
-endif ()
-if (SWIG_LUA)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/lua/*.cxx)
-endif ()
-if (SWIG_GO32)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/go32/*.cxx)
-endif ()
-if (SWIG_GO64)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/go64/*.cxx)
-endif ()
-if (SWIG_JS_JSC)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/js/jsc/*.cxx)
-endif ()
-if (SWIG_JS_V8)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/js/v8/*.cxx)
-endif ()
-if (SWIG_JS_NODE)
-set (libztSwigWrapperSrc ${LIBZT_SRC_DIR}/js/node/*.cxx)
-endif ()
-
 file (GLOB ztcoreSrcGlob
 	${ZTO_SRC_DIR}/node/*.cpp
 	${ZTO_SRC_DIR}/osdep/OSUtils.cpp
@@ -301,7 +270,7 @@ file (GLOB libminiupnpcSrcGlob
     ${ZTO_SRC_DIR}/ext/miniupnpc/upnperrors.c
     ${ZTO_SRC_DIR}/ext/miniupnpc/upnpreplyparse.c)
 
-file (GLOB libztSrcGlob ${LIBZT_SRC_DIR}/*.cpp ${libztSwigWrapperSrc})
+file (GLOB libztSrcGlob ${LIBZT_SRC_DIR}/*.cpp)
 
 if (UNIX)
 	set (LWIP_PORT_DIR ${PROJ_DIR}/ext/lwip-contrib/ports/unix/port)
@@ -321,8 +290,7 @@ file (GLOB lwipSrcGlob
 list(REMOVE_ITEM lwipSrcGlob ${LWIP_SRC_DIR}/netif/slipif.c)
 
 # header globs for xcode frameworks
-file (GLOB frameworkPrivateHeaderGlob include/ZeroTier.h include/ZeroTierConstants.h)
+file (GLOB frameworkPublicHeaderGlob include/ZeroTierSockets.h)
-file (GLOB frameworkPublicHeaderGlob include/Xcode-Bridging-Header.h)
 file (GLOB frameworkHeaderGlob ${frameworkPublicHeaderGlob} ${frameworkPrivateHeaderGlob})
 
 # -----------------------------------------------------------------------------
@@ -457,24 +425,6 @@ set_target_properties (ztcore PROPERTIES
     OUTPUT_NAME ztcore
     LIBRARY_OUTPUT_DIRECTORY ${INTERMEDIATE_LIBRARY_OUTPUT_PATH})
 
-# libnatpmp.a
-#add_library (natpmp STATIC $<TARGET_OBJECTS:libnatpmp_obj>)
-#set_target_properties (natpmp PROPERTIES
-#    OUTPUT_NAME natpmp
-#    LIBRARY_OUTPUT_DIRECTORY ${INTERMEDIATE_LIBRARY_OUTPUT_PATH})
-
-# libminiupnpc.a
-#add_library (miniupnpc STATIC $<TARGET_OBJECTS:miniupnpc_obj>)
-#set_target_properties (miniupnpc PROPERTIES
-#    OUTPUT_NAME miniupnpc
-#    LIBRARY_OUTPUT_DIRECTORY ${INTERMEDIATE_LIBRARY_OUTPUT_PATH})
-
-# liblwip.a
-#add_library (lwip STATIC $<TARGET_OBJECTS:lwip_obj>)
-#set_target_properties (lwip PROPERTIES
-#    OUTPUT_NAME lwip
-#    LIBRARY_OUTPUT_DIRECTORY ${INTERMEDIATE_LIBRARY_OUTPUT_PATH})
-
 # libzt.a
 add_library (${STATIC_LIB_NAME} STATIC $<TARGET_OBJECTS:libzt_obj>
 $<TARGET_OBJECTS:zto_obj>
@@ -540,6 +490,8 @@ endif ()
 # -----------------------------------------------------------------------------
 
 if (SHOULD_BUILD_TESTS)
+	add_executable (earthtest ${PROJ_DIR}/examples/cpp/earthtest.cpp)
+	target_link_libraries(earthtest ${STATIC_LIB_NAME})
 	add_executable (adhoc ${PROJ_DIR}/examples/cpp/adhoc.cpp)
 	target_link_libraries(adhoc ${STATIC_LIB_NAME})
 	add_executable (comprehensive ${PROJ_DIR}/examples/cpp/comprehensive.cpp)
@@ -554,9 +506,7 @@ endif ()
 # |                                  INSTALL                                  |
 # -----------------------------------------------------------------------------
 
-set (PUBLIC_ZT_HEADERS
+set (PUBLIC_ZT_HEADERS ${PROJECT_SOURCE_DIR}/include/ZeroTierSockets.h)
-    ${PROJECT_SOURCE_DIR}/include/ZeroTier.h
-    ${PROJECT_SOURCE_DIR}/include/ZeroTierConstants.h)
 
 set_target_properties(${STATIC_LIB_NAME} PROPERTIES PUBLIC_HEADER "${PUBLIC_ZT_HEADERS}")
 install (TARGETS ${STATIC_LIB_NAME}

README.md

@@ -1,3 +1,9 @@
+---
+title: README
+created: '2020-04-17T17:19:44.552Z'
+modified: '2020-05-02T00:31:45.563Z'
+---
+
 # ZeroTier SDK
 Connect physical devices, virtual devices, and application instances as if everything is on a single LAN.
 ***
@@ -12,15 +18,6 @@ The ZeroTier SDK brings your network into userspace. We've paired our network hy
 
 ### Downloads and examples: [download.zerotier.com/dist/sdk](https://download.zerotier.com/dist/sdk)
 
-### Homebrew
-
-```
-brew install libzt
-clang++ -o yourApp yourApp.cpp -lzt; ./yourApp
-```
-
-### 
-
 ### Building from source
 
 To build both `release` and `debug` libraries for only your host's architecture use `make host`. Or optionally `make host_release` for release only. To build everything including things like iOS frameworks, Android packages, etc, use `make all`. Possible build targets can be seen by using `make list`. Resultant libraries will be placed in `./lib`, test and example programs will be placed in `./bin`:
@@ -67,7 +64,7 @@ The next few sections explain how to use the network control interface portion o
 At this stage, if a cryptographic identity for this node does not already exist on your local storage medium, it will generate a new one and store it, the node's address (commonly referred to as `nodeId`) will be derived from this identity and will be presented to you upon receiving the `ZTS_EVENT_NODE_ONLINE` shown below. The first argument `path` is a path where you will direct ZeroTier to store its automatically-generated cryptographic identity files (`identity.public` and `identity.secret`), these files are your keys to communicating on the network. Keep them safe and keep them unique. If any two nodes are online using the same identities you will have a bad time. The second argument `userCallbackFunc` is a function that you specify to handle all generated events for the life of your program, see below:
 
 ```
-#include "ZeroTier.h"
+#include "ZeroTierSockets.h"
 
 ...
 
@@ -102,8 +99,8 @@ For more complete examples see `./examples/`
 After calling `zts_start()` you will receive one or more of the following events:
 
 ```
-ZTS_EVENT_NODE_OFFLINE
 ZTS_EVENT_NODE_ONLINE
+ZTS_EVENT_NODE_OFFLINE
 ZTS_EVENT_NODE_DOWN
 ZTS_EVENT_NODE_IDENTITY_COLLISION
 ZTS_EVENT_NODE_UNRECOVERABLE_ERROR
@@ -127,7 +124,7 @@ Joining a ZeroTier virtual network is as easy as calling `zts_join(uint64_t netw
 ```
 ZTS_EVENT_NETWORK_NOT_FOUND
 ZTS_EVENT_NETWORK_CLIENT_TOO_OLD
-ZTS_EVENT_NETWORK_REQUESTING_CONFIG
+ZTS_EVENT_NETWORK_REQ_CONFIG
 ZTS_EVENT_NETWORK_OK
 ZTS_EVENT_NETWORK_ACCESS_DENIED
 ZTS_EVENT_NETWORK_READY_IP4
@@ -153,7 +150,7 @@ One can use `zts_get_peer_status(uint64_t peerId)` to query the current reachabi
 
 # Event handling
 
-As mentioned in previous sections, the control API works by use of non-blocking calls and the generation of a few dozen different event types. Depending on the type of event there may be additional contextual information attached to the `zts_callback_msg` object that you can use. This contextual information will be housed in one of the following structures which are defined in `include/ZeroTier.h`:
+As mentioned in previous sections, the control API works by use of non-blocking calls and the generation of a few dozen different event types. Depending on the type of event there may be additional contextual information attached to the `zts_callback_msg` object that you can use. This contextual information will be housed in one of the following structures which are defined in `include/ZeroTierSockets.h`:
 
 ```
 struct zts_callback_msg
@@ -194,9 +191,9 @@ ZTS_EVENT_ADDR_NEW_IP6 --- addr=fda0:9acf:233:e4b0:7099:9309:4c9b:c3c7
 ZTS_EVENT_NODE_ONLINE, node=c4c7ba3cf
 ZTS_EVENT_NETWORK_READY_IP4 --- network=a09acf023be465c1
 ZTS_EVENT_NETWORK_READY_IP6 --- network=a09acf023be465c1
-ZTS_EVENT_PEER_P2P --- node=74d0f5e89d
+ZTS_EVENT_PEER_DIRECT --- node=74d0f5e89d
-ZTS_EVENT_PEER_P2P --- node=9d219039f3
+ZTS_EVENT_PEER_DIRECT --- node=9d219039f3
-ZTS_EVENT_PEER_P2P --- node=a09acf0233
+ZTS_EVENT_PEER_DIRECT --- node=a09acf0233
 ```
 
 ## Node Events
@@ -219,7 +216,7 @@ These events pertain to the state of the indicated network. This event type will
 ```
 ZTS_EVENT_NETWORK_NOT_FOUND
 ZTS_EVENT_NETWORK_CLIENT_TOO_OLD
-ZTS_EVENT_NETWORK_REQUESTING_CONFIG
+ZTS_EVENT_NETWORK_REQ_CONFIG
 ZTS_EVENT_NETWORK_OK
 ZTS_EVENT_NETWORK_ACCESS_DENIED
 ZTS_EVENT_NETWORK_READY_IP4
@@ -234,7 +231,7 @@ ZTS_EVENT_NETWORK_DOWN
 These events are triggered when the reachability status of a peer has changed, this can happen at any time. This event type will arrive with a `zts_peer_details` object for additional context. Additionally, one can query the status of the network with `zts_get_peer_status(uint64_t peerId)`, this will **return** the status as an integer value only.
 
 ```
-ZTS_EVENT_PEER_P2P
+ZTS_EVENT_PEER_DIRECT
 ZTS_EVENT_PEER_RELAY
 ZTS_EVENT_PEER_UNREACHABLE
 ```
@@ -273,13 +270,12 @@ ZTS_EVENT_ADDR_REMOVED_IP6
 
 # Error handling
 
-Calling a `zts_*` function will result in one of the following return codes. Only when `ZTS_ERR` is returned will `zts_errno` be set. Its values closely mirror those used in standard socket interfaces and are defined in `./ext/lwip/src/include/lwip/errno.h`.
+Calling a `zts_*` function will result in one of the following return codes. Only when `ZTS_ERR` is returned will `zts_errno` be set. Its values closely mirror those used in standard socket interfaces and are defined in `include/ZeroTierSockets.h`.
 
  - `ZTS_ERR_OK`: No error
- - `ZTS_ERR`: Error (see `zts_errno`for more information)
+ - `ZTS_ERR_SOCKET`: Socket error (see `zts_errno`for more information)
- - `ZTS_ERR_INVALID_ARG`: An argument provided is invalid.
+ - `ZTS_ERR_SERVICE`: General ZeroTier internal error. Maybe you called something out of order?
- - `ZTS_ERR_SERVICE`: ZT is not yet initialized. Try again.
+ - `ZTS_ERR_ARG`: An argument provided is invalid.
- - `ZTS_ERR_INVALID_OP`: Operation is not permitted (Doesn't make sense in this state).
  - `ZTS_ERR_NO_RESULT`: Call succeeded but no result was available. Not necessarily an error.
  - `ZTS_ERR_GENERAL`: General internal failure. Consider filing a bug report.
 
@@ -326,8 +322,9 @@ zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
 
 If you are calling a host OS function, use your host OS's constants (and structures!):
 ```
-inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
+inet_ntop(AF_INET6, &(in6->sin6_addr), ...);         (CORRECT)
-inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
+inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...);     (INCORRECT)
+zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (CORRECT)
 ```
 
 If you are calling a host OS function but passing a `zts_*` structure, this can work sometimes but you should take care to pass the correct host OS constants:
@@ -361,7 +358,7 @@ If the information in those sections hasn't helped, there are a couple of ways t
 
 2) If you believe your problem is in the network stack you can manually enable debug traces for individual modules in `src/lwipopts.h`. Toggle the `*_DEBUG` types from `LWIP_DBG_OFF` to `LWIP_DBG_ON`. And then rebuild. This will come with a significant performance cost.
 
-3) Enabling network stack statistics. This is useful if you want to monitor the stack's receipt and handling of traffic as well as internal things like memory allocations and cache hits. Protocol and service statistics are available in debug builds of `libzt`. These statistics are detailed fully in the section of `include/ZeroTier.h` that is guarded by `LWIP_STATS`. The protocol constants are defined in `include/ZeroTierConstants.h`. An example usage is as follows:
+3) Enabling network stack statistics. This is useful if you want to monitor the stack's receipt and handling of traffic as well as internal things like memory allocations and cache hits. Protocol and service statistics are available in debug builds of `libzt`. These statistics are detailed fully in the section of `include/ZeroTierSockets.h` that is guarded by `LWIP_STATS`.
 
   ```
   struct zts_stats_proto stats;
@@ -402,7 +399,7 @@ ZTS_EVENT_NETIF_LINK_DOWN
 
 # Network controller mode
 
-The library form of ZeroTier can act as a network controller and in `libzt` this is controlled via the `zts_controller_*` API calls specified in `include/ZeroTier.h`. Currently controller mode is not available in the `iOS` and `macOS` framework builds.
+The library form of ZeroTier can act as a network controller and in `libzt` this is controlled via the `zts_controller_*` API calls specified in `include/ZeroTierSockets.h`. Currently controller mode is not available in the `iOS` and `macOS` framework builds.
 
 # Multipath
 

dist.sh

@@ -535,15 +535,10 @@ package_everything()
     # Source
     mkdir -p $PROD_DIR/src
     cp src/*.cpp src/*.hpp src/*.c src/*.h $PROD_DIR/src
-    # Documentation
-    mkdir -p $PROD_DIR/reference
-    # Copy the errno header from lwIP for customer reference
-    cp ext/lwip/src/include/lwip/errno.h $PROD_DIR/reference
     cp $(pwd)/README.pdf $PROD_DIR/README.pdf
     # Header(s)
     mkdir -p $PROD_DIR/include
     cp $(pwd)/include/*.h $PROD_DIR/include
-    cp $(pwd)/ext/ZeroTierOne/include/ZeroTierOne.h $PROD_DIR/include
     # Libraries
     mkdir -p $PROD_DIR/lib
     cp -r $(pwd)/products/$1/* $PROD_DIR/lib

examples/cpp/adhoc.cpp

@@ -17,8 +17,9 @@
  *   your network, otherwise nothing will happen. This can be done manually or via
  *   our web API: https://my.zerotier.com/help/api
  *
- * - An exception to the above rule is if you are using an Ad-hoc network, it has no 
+ * - Exceptions to the above rule are:
- *   controller and therefore requires no authorization.
+ *    1) Joining a public network (such as "earth")
+ *    2) Joining an Ad-hoc network, (no controller and therefore requires no authorization.)
  *
  *
  *   ESTABLISHING A CONNECTION:
@@ -50,28 +51,27 @@
  *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
  *                returned by these functions can be any of the following:
  *
- *      [ 0] ZTS_ERR_OK          - No error.
+ *      ZTS_ERR_OK            0 // No error
- *      [-1] ZTS_ERR             - Error (see zts_errno for more information).
+ *      ZTS_ERR_SOCKET       -1 // Socket error, see zts_errno
- *      [-2] ZTS_ERR_INVALID_ARG - An argument provided is invalid.
+ *      ZTS_ERR_SERVICE      -2 // You probably did something at the wrong time
- *      [-3] ZTS_ERR_SERVICE     - ZT is not yet initialized. Try again.
+ *      ZTS_ERR_ARG          -3 // Invalid argument
- *      [-4] ZTS_ERR_INVALID_OP  - Operation is not permitted (Doesn't make sense in this state).
+ *      ZTS_ERR_NO_RESULT    -4 // No result (not necessarily an error)
- *      [-5] ZTS_ERR_NO_RESULT   - Call succeeded but no result was available. Not always an error.
+ *      ZTS_ERR_GENERAL      -5 // Consider filing a bug report
- *      [-6] ZTS_ERR_GENERAL     - General internal failure. Consider filing a bug report.
  *
  *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
  *               Errors returned by these functions can be the same as the above. With
  *               the added possibility of zts_errno being set. Much like standard
  *               errno this will provide a more specific reason for an error's occurrence.
- *               These error values are defined in: libzt/ext/lwip/src/include/lwip/errno.h
+ *               See ZeroTierSockets.h for values.
- *               and closely map to standard Linux error values.
  *
  *
  *   API COMPATIBILITY WITH HOST OS:
  *
- * - Since libzt re-implements a socket API probably very similar to your host OS's own
+ * - While the ZeroTier socket interface can coexist with your host OS's own interface in
- *   API it may be tempting to mix and match host OS structures and functions with those
+ *   the same file with no type and naming conflicts, try not to mix and match host
- *   of libzt. This may work on occasion, but you are tempting fate, so here are a few
+ *   OS/libzt structures, functions, or constants. It may look similar and may even work
- *   guidelines:
+ *   some of the time but there enough differences that it will cause headaches. Here
+ *   are a few guidelines:
  *
  *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
  *             
@@ -82,55 +82,23 @@
  *
  *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
  *             ...
- *          zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *          zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
  *
- *   If you are calling a host OS function, use your host OS's constants (and structures!):
- *       
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
- *          inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
- *
- *   If you are calling a host OS function but passing a zts_* structure, this can
- *       work sometimes but you should take care to pass the correct host OS constants:
- *
- *          struct zts_sockaddr_in6 in6;
- *             ...
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), dstStr, INET6_ADDRSTRLEN);
  */
 
 #include <stdio.h>
-#include <string.h>
-#include <string>
-#include <inttypes.h>
 #include <stdlib.h>
 
-#if defined(_WIN32)
+#include "ZeroTierSockets.h"
-#include <WinSock2.h>
-#include <stdint.h>
-#else
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#endif
-
-void delay_ms(long ms)
-{
-#if defined(_WIN32)
-	Sleep(ms);
-#else
-	usleep(ms*1000);
-#endif
-}
-
-#include "ZeroTier.h"
 
 bool nodeReady = false;
 bool networkReady = false;
 
 // Example callbacks
-void myZeroTierEventCallback(struct zts_callback_msg *msg)
+void myZeroTierEventCallback(void *msgPtr)
 {
+	struct zts_callback_msg *msg = (struct zts_callback_msg *)msgPtr;
+
 	// Node events
 	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
 		printf("ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
@@ -146,8 +114,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETWORK_NOT_FOUND --- Are you sure %llx is a valid network?\n",
 			msg->network->nwid);
 	}
-	if (msg->eventCode == ZTS_EVENT_NETWORK_REQUESTING_CONFIG) {
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQ_CONFIG) {
-		printf("ZTS_EVENT_NETWORK_REQUESTING_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+		printf("ZTS_EVENT_NETWORK_REQ_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
 	} 
 	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
 		printf("ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
@@ -174,23 +142,22 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETIF_DOWN --- network=%llx, mac=%llx\n", 
 			msg->netif->nwid,
 			msg->netif->mac);
-		
 		networkReady = true;
 	}
 
 	// Address events
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP6 --- Join %llx and ping me at %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 
 	// Peer events
-	// If you don't recognize the peer ID, don't panic, this is most likely one of our root servers
+	// Don't worry if you don't recognize a peer ID, it's most likely our infrastructure
-	if (msg->eventCode == ZTS_EVENT_PEER_P2P) {
+	if (msg->eventCode == ZTS_EVENT_PEER_DIRECT) {
-		printf("ZTS_EVENT_PEER_P2P --- There is now a direct path to peer %llx\n",
+		printf("ZTS_EVENT_PEER_DIRECT --- There is now a direct path to peer %llx\n",
 			msg->peer->address);
 	}
 	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
@@ -238,14 +205,14 @@ int main(int argc, char **argv)
 	uint64_t adhoc_nwid = zts_generate_adhoc_nwid_from_range(adhocStartPort, adhocEndPort);
 	int err = ZTS_ERR_OK;
 
-	zts_set_network_caching(false);
+	zts_allow_network_caching(false);
 
 	if((err = zts_start(argv[1], &myZeroTierEventCallback, ztServicePort)) != ZTS_ERR_OK) {
 		printf("Unable to start service, error = %d. Exiting.\n", err);
 		exit(1);
 	}
 	printf("Waiting for node to come online...\n");
-	while (!nodeReady) { delay_ms(50); }
+	while (!nodeReady) { zts_delay_ms(50); }
 	printf("This node's identity is stored in %s\n", argv[1]);
 
 	if((err = zts_join(adhoc_nwid)) != ZTS_ERR_OK) {
@@ -253,12 +220,12 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 	printf("Joining network %llx\n", adhoc_nwid);
-	while (!networkReady) { delay_ms(50); }
+	while (!networkReady) { zts_delay_ms(50); }
 
 	// Idle and just show callback events, stack statistics, etc
 
 	printf("Node will now idle...\n");
-	while (true) { delay_ms(1000); }
+	while (true) { zts_delay_ms(1000); }
 
 	// Shut down service and stack threads
 

examples/cpp/client.cpp

@@ -3,38 +3,20 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <string>
-#include <inttypes.h>
 
-#if defined(_WIN32)
+#include "ZeroTierSockets.h"
-#include <WinSock2.h>
-#include <stdint.h>
-#else
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#endif
-
-void delay_ms(long ms)
-{
-#if defined(_WIN32)
-	Sleep(ms);
-#else
-	usleep(ms*1000);
-#endif
-}
-
-#include "ZeroTier.h"
 
 bool nodeReady = false;
 bool networkReady = false;
 
 // Example callbacks
-void myZeroTierEventCallback(struct zts_callback_msg *msg)
+void myZeroTierEventCallback(void *msgPtr)
 {
+	struct zts_callback_msg *msg = (struct zts_callback_msg *)msgPtr;
+
 	// Node events
 	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
 		printf("ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
@@ -53,8 +35,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETWORK_NOT_FOUND --- Are you sure %llx is a valid network?\n",
 			msg->network->nwid);
 	}
-	if (msg->eventCode == ZTS_EVENT_NETWORK_REQUESTING_CONFIG) {
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQ_CONFIG) {
-		printf("ZTS_EVENT_NETWORK_REQUESTING_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+		printf("ZTS_EVENT_NETWORK_REQ_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
 	} 
 	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
 		printf("ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
@@ -76,36 +58,36 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 
 	// Address events
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP4 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP6 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP4 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP6 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 	// Peer events
-	if (msg->eventCode == ZTS_EVENT_PEER_P2P) {
+	if (msg->eventCode == ZTS_EVENT_PEER_DIRECT) {
-		printf("ZTS_EVENT_PEER_P2P --- node=%llx\n", msg->peer->address);
+		printf("ZTS_EVENT_PEER_DIRECT --- node=%llx\n", msg->peer->address);
 		// A direct path is known for nodeId
 	}
 	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
@@ -127,8 +109,9 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *   your network, otherwise nothing will happen. This can be done manually or via
  *   our web API: https://my.zerotier.com/help/api
  *
- * - An exception to the above rule is if you are using an Ad-hoc network, it has no 
+ * - Exceptions to the above rule are:
- *   controller and therefore requires no authorization.
+ *    1) Joining a public network (such as "earth")
+ *    2) Joining an Ad-hoc network, (no controller and therefore requires no authorization.)
  *
  *
  *   ESTABLISHING A CONNECTION:
@@ -160,28 +143,27 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
  *                returned by these functions can be any of the following:
  *
- *      [ 0] ZTS_ERR_OK          - No error.
+ *      ZTS_ERR_OK            0 // No error
- *      [-1] ZTS_ERR             - Error (see zts_errno for more information).
+ *      ZTS_ERR_SOCKET       -1 // Socket error, see zts_errno
- *      [-2] ZTS_ERR_INVALID_ARG - An argument provided is invalid.
+ *      ZTS_ERR_SERVICE      -2 // You probably did something at the wrong time
- *      [-3] ZTS_ERR_SERVICE     - ZT is not yet initialized. Try again.
+ *      ZTS_ERR_ARG          -3 // Invalid argument
- *      [-4] ZTS_ERR_INVALID_OP  - Operation is not permitted (Doesn't make sense in this state).
+ *      ZTS_ERR_NO_RESULT    -4 // No result (not necessarily an error)
- *      [-5] ZTS_ERR_NO_RESULT   - Call succeeded but no result was available. Not always an error.
+ *      ZTS_ERR_GENERAL      -5 // Consider filing a bug report
- *      [-6] ZTS_ERR_GENERAL     - General internal failure. Consider filing a bug report.
  *
  *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
  *               Errors returned by these functions can be the same as the above. With
  *               the added possibility of zts_errno being set. Much like standard
  *               errno this will provide a more specific reason for an error's occurrence.
- *               These error values are defined in: libzt/ext/lwip/src/include/lwip/errno.h
+ *               See ZeroTierSockets.h for values.
- *               and closely map to standard Linux error values.
  *
  *
  *   API COMPATIBILITY WITH HOST OS:
  *
- * - Since libzt re-implements a socket API probably very similar to your host OS's own
+ * - While the ZeroTier socket interface can coexist with your host OS's own interface in
- *   API it may be tempting to mix and match host OS structures and functions with those
+ *   the same file with no type and naming conflicts, try not to mix and match host
- *   of libzt. This may work on occasion, but you are tempting fate, so here are a few
+ *   OS/libzt structures, functions, or constants. It may look similar and may even work
- *   guidelines:
+ *   some of the time but there enough differences that it will cause headaches. Here
+ *   are a few guidelines:
  *
  *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
  *             
@@ -192,19 +174,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *
  *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
  *             ...
- *          zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *          zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
  *
- *   If you are calling a host OS function, use your host OS's constants (and structures!):
- *       
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
- *          inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
- *
- *   If you are calling a host OS function but passing a zts_* structure, this can
- *       work sometimes but you should take care to pass the correct host OS constants:
- *
- *          struct zts_sockaddr_in6 in6;
- *             ...
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), dstStr, INET6_ADDRSTRLEN);
  */
 
 int main(int argc, char **argv) 
@@ -220,11 +191,11 @@ int main(int argc, char **argv)
 	int ztServicePort = atoi(argv[5]); // Port ZT uses to send encrypted UDP packets to peers (try something like 9994)
 
 	struct zts_sockaddr_in in4;
-	in4.sin_port = htons(remotePort);
+	in4.sin_port = zts_htons(remotePort);
 #if defined(_WIN32)
-		in4.sin_addr.S_addr = inet_addr(remoteAddr.c_str());;
+	in4.sin_addr.S_addr = zts_inet_addr(remoteAddr.c_str());
 #else
-		in4.sin_addr.s_addr = inet_addr(remoteAddr.c_str());;
+	in4.sin_addr.s_addr = zts_inet_addr(remoteAddr.c_str());
 #endif
 	in4.sin_family = ZTS_AF_INET;
 
@@ -237,7 +208,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 	printf("Waiting for node to come online...\n");
-	while (!nodeReady) { delay_ms(50); }
+	while (!nodeReady) { zts_delay_ms(50); }
 	printf("This node's identity is stored in %s\n", argv[1]);
 
 	if((err = zts_join(nwid)) != ZTS_ERR_OK) {
@@ -246,7 +217,7 @@ int main(int argc, char **argv)
 	}
 	printf("Joining network %llx\n", nwid);
 	printf("Don't forget to authorize this device in my.zerotier.com or the web API!\n");
-	while (!networkReady) { delay_ms(50); }
+	while (!networkReady) { zts_delay_ms(50); }
 
 	// Socket-like API example
 
@@ -262,7 +233,7 @@ int main(int argc, char **argv)
 	// Retries are often required since ZT uses transport-triggered links (explained above)
 	for (;;) {
 		printf("Connecting to remote host...\n");
-		if ((err = zts_connect(fd, (const struct sockaddr *)&in4, sizeof(in4))) < 0) {
+		if ((err = zts_connect(fd, (const struct zts_sockaddr *)&in4, sizeof(in4))) < 0) {
 			printf("Error connecting to remote host (fd=%d, ret=%d, zts_errno=%d). Trying again.\n",
 				fd, err, zts_errno);
 			zts_close(fd);
@@ -271,7 +242,7 @@ int main(int argc, char **argv)
 				printf("Error creating ZeroTier socket (fd=%d, zts_errno=%d). Exiting.\n", fd, zts_errno);
 				exit(1);
 			}
-			delay_ms(250);
+			zts_delay_ms(250);
 		} 
 		else {
 			printf("Connected.\n");

examples/cpp/comprehensive.cpp

@@ -17,8 +17,9 @@
  *   your network, otherwise nothing will happen. This can be done manually or via
  *   our web API: https://my.zerotier.com/help/api
  *
- * - An exception to the above rule is if you are using an Ad-hoc network, it has no 
+ * - Exceptions to the above rule are:
- *   controller and therefore requires no authorization.
+ *    1) Joining a public network (such as "earth")
+ *    2) Joining an Ad-hoc network, (no controller and therefore requires no authorization.)
  *
  *
  *   ESTABLISHING A CONNECTION:
@@ -50,28 +51,27 @@
  *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
  *                returned by these functions can be any of the following:
  *
- *      [ 0] ZTS_ERR_OK          - No error.
+ *      ZTS_ERR_OK            0 // No error
- *      [-1] ZTS_ERR             - Error (see zts_errno for more information).
+ *      ZTS_ERR_SOCKET       -1 // Socket error, see zts_errno
- *      [-2] ZTS_ERR_INVALID_ARG - An argument provided is invalid.
+ *      ZTS_ERR_SERVICE      -2 // You probably did something at the wrong time
- *      [-3] ZTS_ERR_SERVICE     - ZT is not yet initialized. Try again.
+ *      ZTS_ERR_ARG          -3 // Invalid argument
- *      [-4] ZTS_ERR_INVALID_OP  - Operation is not permitted (Doesn't make sense in this state).
+ *      ZTS_ERR_NO_RESULT    -4 // No result (not necessarily an error)
- *      [-5] ZTS_ERR_NO_RESULT   - Call succeeded but no result was available. Not always an error.
+ *      ZTS_ERR_GENERAL      -5 // Consider filing a bug report
- *      [-6] ZTS_ERR_GENERAL     - General internal failure. Consider filing a bug report.
  *
  *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
  *               Errors returned by these functions can be the same as the above. With
  *               the added possibility of zts_errno being set. Much like standard
  *               errno this will provide a more specific reason for an error's occurrence.
- *               These error values are defined in: libzt/ext/lwip/src/include/lwip/errno.h
+ *               See ZeroTierSockets.h for values.
- *               and closely map to standard Linux error values.
  *
  *
  *   API COMPATIBILITY WITH HOST OS:
  *
- * - Since libzt re-implements a socket API probably very similar to your host OS's own
+ * - While the ZeroTier socket interface can coexist with your host OS's own interface in
- *   API it may be tempting to mix and match host OS structures and functions with those
+ *   the same file with no type and naming conflicts, try not to mix and match host
- *   of libzt. This may work on occasion, but you are tempting fate, so here are a few
+ *   OS/libzt structures, functions, or constants. It may look similar and may even work
- *   guidelines:
+ *   some of the time but there enough differences that it will cause headaches. Here
+ *   are a few guidelines:
  *
  *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
  *             
@@ -82,55 +82,23 @@
  *
  *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
  *             ...
- *          zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *          zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
  *
- *   If you are calling a host OS function, use your host OS's constants (and structures!):
- *       
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
- *          inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
- *
- *   If you are calling a host OS function but passing a zts_* structure, this can
- *       work sometimes but you should take care to pass the correct host OS constants:
- *
- *          struct zts_sockaddr_in6 in6;
- *             ...
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), dstStr, INET6_ADDRSTRLEN);
  */
 
 #include <stdio.h>
-#include <string.h>
-#include <string>
-#include <inttypes.h>
 #include <stdlib.h>
 
-#if defined(_WIN32)
+#include "ZeroTierSockets.h"
-#include <WinSock2.h>
-#include <stdint.h>
-#else
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#endif
-
-void delay_ms(long ms)
-{
-#if defined(_WIN32)
-	Sleep(ms);
-#else
-	usleep(ms*1000);
-#endif
-}
-
-#include "ZeroTier.h"
 
 bool nodeReady = false;
 bool networkReady = false;
 
 // Example callbacks
-void myZeroTierEventCallback(struct zts_callback_msg *msg)
+void myZeroTierEventCallback(void *msgPtr)
 {
+	struct zts_callback_msg *msg = (struct zts_callback_msg *)msgPtr;
+
 	// Node events
 	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
 		printf("ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
@@ -149,8 +117,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETWORK_NOT_FOUND --- Are you sure %llx is a valid network?\n",
 			msg->network->nwid);
 	}
-	if (msg->eventCode == ZTS_EVENT_NETWORK_REQUESTING_CONFIG) {
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQ_CONFIG) {
-		printf("ZTS_EVENT_NETWORK_REQUESTING_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+		printf("ZTS_EVENT_NETWORK_REQ_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
 	} 
 	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
 		printf("ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
@@ -182,43 +150,42 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETIF_DOWN --- network=%llx, mac=%llx\n", 
 			msg->netif->nwid,
 			msg->netif->mac);
-		
 		networkReady = true;
 	}
 
 	// Address events
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP4 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP6 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP4 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP6 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 
 	// Peer events
-	if (msg->eventCode == ZTS_EVENT_PEER_P2P) {
+	if (msg->eventCode == ZTS_EVENT_PEER_DIRECT) {
-		printf("ZTS_EVENT_PEER_P2P --- node=%llx\n", msg->peer->address);
+		printf("ZTS_EVENT_PEER_DIRECT --- node=%llx\n", msg->peer->address);
 		// A direct path is known for nodeId
 	}
 	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
@@ -238,17 +205,17 @@ void printPeerDetails(struct zts_peer_details *pd)
 		pd->pathCount);
 	// Print all known paths for each peer
 	for (unsigned int j=0; j<pd->pathCount; j++) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		int port = 0;
-		struct sockaddr *sa = (struct sockaddr *)&(pd->paths[j].address);
+		struct zts_sockaddr *sa = (struct zts_sockaddr *)&(pd->paths[j].address);
-		if (sa->sa_family == AF_INET) {
+		if (sa->sa_family == ZTS_AF_INET) { // TODO: Probably broken
 			struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)sa;
-			inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+			zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
-			port = ntohs(in4->sin_port);
+			port = zts_ntohs(in4->sin_port);
 		}
-		if (sa->sa_family == AF_INET6) {
+		if (sa->sa_family == ZTS_AF_INET6) {
 			struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)sa;
-			inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+			zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		}
 		printf("\tpath[%d]=%s, port=%d\n", j, ipstr, port);	
 	}
@@ -261,22 +228,17 @@ void getSinglePeerDetails(uint64_t peerId)
 
 	if (err == ZTS_ERR_OK) {
 		printf("(%d) call succeeded\n", err);
-	} if (err == ZTS_ERR_INVALID_ARG) {
+		printPeerDetails(&pd);
+	} if (err == ZTS_ERR_ARG) {
 		printf("(%d) invalid argument\n", err);
 		return;
 	} if (err == ZTS_ERR_SERVICE) {
-		printf("(%d) error: service is unavailable\n", err);
+		printf("(%d) error: invalid API operation or service error\n", err);
-		return;
-	} if (err == ZTS_ERR_INVALID_OP) {
-		printf("(%d) error: invalid API operation\n", err);
 		return;
 	} if (err == ZTS_ERR_NO_RESULT) {
 		printf("(%d) error: object or result not found\n", err);
 		return;
 	}
-	if (err == 0) { // ZTS_ERR_OK
-		printPeerDetails(&pd);
-	}
 }
 
 // Similar to "zerotier-cli listpeers"
@@ -348,7 +310,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 	printf("Waiting for node to come online...\n");
-	while (!nodeReady) { delay_ms(50); }
+	while (!nodeReady) { zts_delay_ms(50); }
 	printf("This node ID is %llx\n", zts_get_node_id());
 	printf("This node's identity is stored in %s\n", argv[1]);
 
@@ -358,7 +320,7 @@ int main(int argc, char **argv)
 	}
 	printf("Joining network %llx\n", nwid);
 	printf("Don't forget to authorize this device in my.zerotier.com or the web API!\n");
-	while (!networkReady) { delay_ms(50); }
+	while (!networkReady) { zts_delay_ms(50); }
 
 	// Get multiple peer's details 
 	getAllPeerDetails();
@@ -379,7 +341,7 @@ int main(int argc, char **argv)
 	// Idle and just show callback events, stack statistics, etc
 
 	while (true) {
-		delay_ms(1000);
+		zts_delay_ms(1000);
 		status = zts_get_node_status();
 		printf("zts_get_node_status()=%d\n", status);
 		display_stack_stats();

examples/cpp/earthtest.cpp

@@ -0,0 +1,188 @@
+/**
+ * libzt API example
+ *
+ * Pingable node joined to public ZT network "earth"
+ */
+
+/**
+ *
+ *   IDENTITIES and AUTHORIZATION:
+ *
+ * - Upon the first execution of this code, a new identity will be generated and placed in
+ *   the location given in the first argument to zts_start(path, ...). If you accidentally
+ *   duplicate the identity files and use them simultaneously in a different node instance
+ *   you will experience undefined behavior and it is likely nothing will work.
+ * 
+ * - You must authorize the node ID provided by the ZTS_EVENT_NODE_ONLINE callback to join
+ *   your network, otherwise nothing will happen. This can be done manually or via
+ *   our web API: https://my.zerotier.com/help/api
+ *
+ * - Exceptions to the above rule are:
+ *    1) Joining a public network (such as "earth")
+ *    2) Joining an Ad-hoc network, (no controller and therefore requires no authorization.)
+ *
+ *
+ *   ESTABLISHING A CONNECTION:
+ *
+ * - Creating a standard socket connection generally works the same as it would using
+ *   an ordinary socket interface, however with libzt there is a subtle difference in
+ *   how connections are established which may cause confusion:
+ *
+ *   The underlying virtual ZT layer creates what are called "transport-triggered links"
+ *   between nodes. That is, links are not established until an attempt to communicate
+ *   with a peer has taken place. The side effect is that the first few packets sent from
+ *   a libzt instance are usually relayed via our free infrastructure and it isn't until a
+ *   root server has passed contact information to both peers that a direct connection will be
+ *   established. Therefore, it is required that multiple connection attempts be undertaken
+ *   when initially communicating with a peer. After a transport-triggered link is
+ *   established libzt will inform you via ZTS_EVENT_PEER_P2P for a specific peer ID. No
+ *   action is required on your part for this callback event.
+ *
+ *   Note: In these initial moments before ZTS_EVENT_PEER_P2P has been received for a
+ *         specific peer, traffic may be slow, jittery and there may be high packet loss.
+ *         This will subside within a couple of seconds.
+ *
+ *
+ *   ERROR HANDLING:
+ *
+ * - libzt's API is actually composed of two categories of functions with slightly
+ *   different error reporting mechanisms.
+ *
+ *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
+ *                returned by these functions can be any of the following:
+ *
+ *      ZTS_ERR_OK            0 // No error
+ *      ZTS_ERR_SOCKET       -1 // Socket error, see zts_errno
+ *      ZTS_ERR_SERVICE      -2 // You probably did something at the wrong time
+ *      ZTS_ERR_ARG          -3 // Invalid argument
+ *      ZTS_ERR_NO_RESULT    -4 // No result (not necessarily an error)
+ *      ZTS_ERR_GENERAL      -5 // Consider filing a bug report
+ *
+ *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
+ *               Errors returned by these functions can be the same as the above. With
+ *               the added possibility of zts_errno being set. Much like standard
+ *               errno this will provide a more specific reason for an error's occurrence.
+ *               See ZeroTierSockets.h for values.
+ *
+ *
+ *   API COMPATIBILITY WITH HOST OS:
+ *
+ * - While the ZeroTier socket interface can coexist with your host OS's own interface in
+ *   the same file with no type and naming conflicts, try not to mix and match host
+ *   OS/libzt structures, functions, or constants. It may look similar and may even work
+ *   some of the time but there enough differences that it will cause headaches. Here
+ *   are a few guidelines:
+ *
+ *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
+ *             
+ *          zts_socket(ZTS_AF_INET6, ZTS_SOCK_DGRAM, 0); (CORRECT)
+ *          zts_socket(AF_INET6, SOCK_DGRAM, 0);         (INCORRECT)
+ *
+ *   If you are calling a zts_* function, use the appropriate zts_* structure:
+ *
+ *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
+ *             ...
+ *          zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "ZeroTierSockets.h"
+
+bool nodeReady = false;
+bool networkReady = false;
+
+// Example callbacks
+void myZeroTierEventCallback(void *msgPtr)
+{
+	struct zts_callback_msg *msg = (struct zts_callback_msg *)msgPtr;
+
+	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
+		printf("ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
+		nodeReady = true;
+	}
+	if (msg->eventCode == ZTS_EVENT_NODE_OFFLINE) {
+		printf("ZTS_EVENT_NODE_OFFLINE --- Check your physical Internet connection, router, firewall, etc. What ports are you blocking?\n");
+		nodeReady = false;
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQ_CONFIG) {
+		printf("ZTS_EVENT_NETWORK_REQ_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+	} 
+	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
+		printf("ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
+			msg->network->nwid);
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_READY_IP6) {
+		printf("ZTS_EVENT_NETWORK_READY_IP6 --- Network config received. IPv6 traffic can now be sent over network %llx\n",
+			msg->network->nwid);
+		networkReady = true;
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_DOWN) {
+		printf("ZTS_EVENT_NETWORK_DOWN --- %llx\n", msg->network->nwid);
+	}
+	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP4) {
+		char ipstr[ZTS_INET_ADDRSTRLEN];
+		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
+		printf("ZTS_EVENT_ADDR_NEW_IP4 --- Join %llx and ping me at %s\n", 
+			msg->addr->nwid, ipstr);
+	}
+	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
+		printf("ZTS_EVENT_ADDR_NEW_IP6 --- Join %llx and ping me at %s\n", 
+			msg->addr->nwid, ipstr);
+	}
+	// Don't worry if you don't recognize a peer ID, it's most likely our infrastructure
+	if (msg->eventCode == ZTS_EVENT_PEER_DIRECT) {
+		printf("ZTS_EVENT_PEER_DIRECT --- There is now a direct path to peer %llx\n",
+			msg->peer->address);
+	}
+	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
+		printf("ZTS_EVENT_PEER_RELAY --- No direct path to peer %llx\n",
+			msg->peer->address);
+	}
+}
+
+int main(int argc, char **argv) 
+{
+	if (argc != 3) {
+		printf("\nlibzt example\n");
+		printf("earthtest <config_file_path> <ztServicePort>\n");
+		exit(0);
+	}
+	int ztServicePort = atoi(argv[2]); // Port ZT uses to send encrypted UDP packets to peers (try something like 9994)
+	
+	int err = ZTS_ERR_OK;
+	zts_allow_network_caching(false);
+
+	if((err = zts_start(argv[1], &myZeroTierEventCallback, ztServicePort)) != ZTS_ERR_OK) {
+		printf("Unable to start service, error = %d. Exiting.\n", err);
+		exit(1);
+	}
+	printf("Waiting for node to come online...\n");
+	while (!nodeReady) { zts_delay_ms(50); }
+	printf("This node's identity is stored in %s\n", argv[1]);
+
+	uint64_t nwid = 0x8056c2e21c000001;
+
+	if((err = zts_join(nwid)) != ZTS_ERR_OK) {
+		printf("Unable to join network, error = %d. Exiting.\n", err);
+		exit(1);
+	}
+	printf("Joining network %llx\n", nwid);
+	while (!networkReady) { zts_delay_ms(50); }
+
+	// Idle and just show callback events, stack statistics, etc
+
+	printf("Node will now idle...\n");
+	while (true) { zts_delay_ms(1000); }
+
+	// Shut down service and stack threads
+
+	zts_stop();
+	return 0;
+}

examples/cpp/server.cpp

@@ -3,38 +3,20 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <string>
-#include <inttypes.h>
 
-#if defined(_WIN32)
+#include "ZeroTierSockets.h"
-#include <WinSock2.h>
-#include <stdint.h>
-#else
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <unistd.h>
-#endif
-
-void delay_ms(long ms)
-{
-#if defined(_WIN32)
-	Sleep(ms);
-#else
-	usleep(ms*1000);
-#endif
-}
-
-#include "ZeroTier.h"
 
 bool nodeReady = false;
 bool networkReady = false;
 
 // Example callbacks
-void myZeroTierEventCallback(struct zts_callback_msg *msg)
+void myZeroTierEventCallback(void *msgPtr)
 {
+	struct zts_callback_msg *msg = (struct zts_callback_msg *)msgPtr;
+
 	// Node events
 	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
 		printf("ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
@@ -53,8 +35,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 		printf("ZTS_EVENT_NETWORK_NOT_FOUND --- Are you sure %llx is a valid network?\n",
 			msg->network->nwid);
 	}
-	if (msg->eventCode == ZTS_EVENT_NETWORK_REQUESTING_CONFIG) {
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQ_CONFIG) {
-		printf("ZTS_EVENT_NETWORK_REQUESTING_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+		printf("ZTS_EVENT_NETWORK_REQ_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
 	} 
 	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
 		printf("ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
@@ -76,36 +58,36 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
 
 	// Address events
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP4 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_NEW_IP6 --- This node's virtual address on network %llx is %s\n", 
 			msg->addr->nwid, ipstr);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP4) {
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg->addr->addr);
-		inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(in4->sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP4 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 	if (msg->eventCode == ZTS_EVENT_ADDR_REMOVED_IP6) {
-		char ipstr[INET6_ADDRSTRLEN];
+		char ipstr[ZTS_INET6_ADDRSTRLEN];
 		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
-		inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ipstr, ZTS_INET6_ADDRSTRLEN);
 		printf("ZTS_EVENT_ADDR_REMOVED_IP6 --- The virtual address %s for this node on network %llx has been removed.\n", 
 			ipstr, msg->addr->nwid);
 	}
 	// Peer events
-	if (msg->eventCode == ZTS_EVENT_PEER_P2P) {
+	if (msg->eventCode == ZTS_EVENT_PEER_DIRECT) {
-		printf("ZTS_EVENT_PEER_P2P --- node=%llx\n", msg->peer->address);
+		printf("ZTS_EVENT_PEER_DIRECT --- node=%llx\n", msg->peer->address);
 		// A direct path is known for nodeId
 	}
 	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
@@ -127,8 +109,9 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *   your network, otherwise nothing will happen. This can be done manually or via
  *   our web API: https://my.zerotier.com/help/api
  *
- * - An exception to the above rule is if you are using an Ad-hoc network, it has no 
+ * - Exceptions to the above rule are:
- *   controller and therefore requires no authorization.
+ *    1) Joining a public network (such as "earth")
+ *    2) Joining an Ad-hoc network, (no controller and therefore requires no authorization.)
  *
  *
  *   ESTABLISHING A CONNECTION:
@@ -160,28 +143,27 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
  *                returned by these functions can be any of the following:
  *
- *      [ 0] ZTS_ERR_OK          - No error.
+ *      ZTS_ERR_OK            0 // No error
- *      [-1] ZTS_ERR             - Error (see zts_errno for more information).
+ *      ZTS_ERR_SOCKET       -1 // Socket error, see zts_errno
- *      [-2] ZTS_ERR_INVALID_ARG - An argument provided is invalid.
+ *      ZTS_ERR_SERVICE      -2 // You probably did something at the wrong time
- *      [-3] ZTS_ERR_SERVICE     - ZT is not yet initialized. Try again.
+ *      ZTS_ERR_ARG          -3 // Invalid argument
- *      [-4] ZTS_ERR_INVALID_OP  - Operation is not permitted (Doesn't make sense in this state).
+ *      ZTS_ERR_NO_RESULT    -4 // No result (not necessarily an error)
- *      [-5] ZTS_ERR_NO_RESULT   - Call succeeded but no result was available. Not always an error.
+ *      ZTS_ERR_GENERAL      -5 // Consider filing a bug report
- *      [-6] ZTS_ERR_GENERAL     - General internal failure. Consider filing a bug report.
  *
  *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
  *               Errors returned by these functions can be the same as the above. With
  *               the added possibility of zts_errno being set. Much like standard
  *               errno this will provide a more specific reason for an error's occurrence.
- *               These error values are defined in: libzt/ext/lwip/src/include/lwip/errno.h
+ *               See ZeroTierSockets.h for values.
- *               and closely map to standard Linux error values.
  *
  *
  *   API COMPATIBILITY WITH HOST OS:
  *
- * - Since libzt re-implements a socket API probably very similar to your host OS's own
+ * - While the ZeroTier socket interface can coexist with your host OS's own interface in
- *   API it may be tempting to mix and match host OS structures and functions with those
+ *   the same file with no type and naming conflicts, try not to mix and match host
- *   of libzt. This may work on occasion, but you are tempting fate, so here are a few
+ *   OS/libzt structures, functions, or constants. It may look similar and may even work
- *   guidelines:
+ *   some of the time but there enough differences that it will cause headaches. Here
+ *   are a few guidelines:
  *
  *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
  *             
@@ -192,19 +174,8 @@ void myZeroTierEventCallback(struct zts_callback_msg *msg)
  *
  *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
  *             ...
- *          zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *          zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
  *
- *   If you are calling a host OS function, use your host OS's constants (and structures!):
- *       
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
- *          inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
- *
- *   If you are calling a host OS function but passing a zts_* structure, this can
- *       work sometimes but you should take care to pass the correct host OS constants:
- *
- *          struct zts_sockaddr_in6 in6;
- *             ...
- *          inet_ntop(AF_INET6, &(in6->sin6_addr), dstStr, INET6_ADDRSTRLEN);
  */
 
 int main(int argc, char **argv) 
@@ -219,11 +190,11 @@ int main(int argc, char **argv)
 	int ztServicePort = atoi(argv[4]); // Port ZT uses to send encrypted UDP packets to peers (try something like 9994)
 	
 	struct zts_sockaddr_in in4, acc_in4;
-	in4.sin_port = htons(serverBindPort);
+	in4.sin_port = zts_htons(serverBindPort);
 #if defined(_WIN32)
-	in4.sin_addr.S_addr = INADDR_ANY;
+	in4.sin_addr.S_addr = ZTS_INADDR_ANY;
 #else
-	in4.sin_addr.s_addr = INADDR_ANY;
+	in4.sin_addr.s_addr = ZTS_INADDR_ANY;
 #endif
 	in4.sin_family = ZTS_AF_INET;
 
@@ -237,7 +208,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 	printf("Waiting for node to come online...\n");
-	while (!nodeReady) { delay_ms(50); }
+	while (!nodeReady) { zts_delay_ms(50); }
 	printf("This node's identity is stored in %s\n", argv[1]);
 
 	if((err = zts_join(nwid)) != ZTS_ERR_OK) {
@@ -246,7 +217,7 @@ int main(int argc, char **argv)
 	}
 	printf("Joining network %llx\n", nwid);
 	printf("Don't forget to authorize this device in my.zerotier.com or the web API!\n");
-	while (!networkReady) { delay_ms(50); }
+	while (!networkReady) { zts_delay_ms(50); }
 
 	// Socket-like API example
 
@@ -256,7 +227,7 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 	printf("Binding...\n");
-	if ((err = zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)) {
+	if ((err = zts_bind(fd, (struct zts_sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)) {
 		printf("Error binding to interface (fd=%d, ret=%d, zts_errno=%d). Exiting.\n", fd, err, zts_errno);
 		exit(1);
 	}
@@ -272,17 +243,17 @@ int main(int argc, char **argv)
 	memset(recvBuf, 0, sizeof(recvBuf));
 
 	while (true) {
-		socklen_t client_addrlen = sizeof(zts_sockaddr_in);
+		zts_socklen_t client_addrlen = sizeof(zts_sockaddr_in);
-		if ((accfd = zts_accept(fd, (struct sockaddr *)&acc_in4, &client_addrlen)) < 0) {
+		if ((accfd = zts_accept(fd, (struct zts_sockaddr *)&acc_in4, &client_addrlen)) < 0) {
 			printf("Error accepting connection (fd=%d, ret=%d, zts_errno=%d). Exiting.\n", fd, err, zts_errno);
 		}
-		socklen_t peer_addrlen = sizeof(struct zts_sockaddr_storage);
+		zts_socklen_t peer_addrlen = sizeof(struct zts_sockaddr_storage);
-		zts_getpeername(accfd, (struct sockaddr*)&acc_in4, &peer_addrlen);
+		zts_getpeername(accfd, (struct zts_sockaddr*)&acc_in4, &peer_addrlen);
 
-		char ipstr[INET_ADDRSTRLEN];
+		char ipstr[ZTS_INET_ADDRSTRLEN];
 		memset(ipstr, 0, sizeof(ipstr));
-		inet_ntop(AF_INET, &(acc_in4.sin_addr), ipstr, INET_ADDRSTRLEN);
+		zts_inet_ntop(ZTS_AF_INET, &(acc_in4.sin_addr), ipstr, ZTS_INET_ADDRSTRLEN);
-		printf("Accepted connection from %s:%d\n", ipstr, ntohs(acc_in4.sin_port));
+		printf("Accepted connection from %s:%d\n", ipstr, zts_ntohs(acc_in4.sin_port));
 
 		printf("Reading message string from client...\n");
 		if((bytes = zts_read(accfd, recvBuf, sizeof(recvBuf))) < 0) {

examples/objective-c/adhoc.m

@@ -0,0 +1,247 @@
+/**
+ * libzt API example
+ *
+ * Specify location of zt.framework and link to standard C++ library:
+ *
+ * clang -lc++ -framework Foundation -F . -framework zt adhoc.m -o adhoc;
+ *
+ * Pingable node joined to controller-less adhoc network with a 6PLANE addressing scheme
+ */
+
+#import <Foundation/Foundation.h>
+
+#import <zt/ZeroTier.h>
+
+#include <arpa/inet.h>
+
+/**
+ *
+ *   IDENTITIES and AUTHORIZATION:
+ *
+ * - Upon the first execution of this code, a new identity will be generated and placed in
+ *   the location given in the first argument to zts_start(path, ...). If you accidentally
+ *   duplicate the identity files and use them simultaneously in a different node instance
+ *   you will experience undefined behavior and it is likely nothing will work.
+ * 
+ * - You must authorize the node ID provided by the ZTS_EVENT_NODE_ONLINE callback to join
+ *   your network, otherwise nothing will happen. This can be done manually or via
+ *   our web API: https://my.zerotier.com/help/api
+ *
+ * - An exception to the above rule is if you are using an Ad-hoc network, it has no 
+ *   controller and therefore requires no authorization.
+ *
+ *
+ *   ESTABLISHING A CONNECTION:
+ *
+ * - Creating a standard socket connection generally works the same as it would using
+ *   an ordinary socket interface, however with libzt there is a subtle difference in
+ *   how connections are established which may cause confusion:
+ *
+ *   The underlying virtual ZT layer creates what are called "transport-triggered links"
+ *   between nodes. That is, links are not established until an attempt to communicate
+ *   with a peer has taken place. The side effect is that the first few packets sent from
+ *   a libzt instance are usually relayed via our free infrastructure and it isn't until a
+ *   root server has passed contact information to both peers that a direct connection will be
+ *   established. Therefore, it is required that multiple connection attempts be undertaken
+ *   when initially communicating with a peer. After a transport-triggered link is
+ *   established libzt will inform you via ZTS_EVENT_PEER_P2P for a specific peer ID. No
+ *   action is required on your part for this callback event.
+ *
+ *   Note: In these initial moments before ZTS_EVENT_PEER_P2P has been received for a
+ *         specific peer, traffic may be slow, jittery and there may be high packet loss.
+ *         This will subside within a couple of seconds.
+ *
+ *
+ *   ERROR HANDLING:
+ *
+ * - libzt's API is actually composed of two categories of functions with slightly
+ *   different error reporting mechanisms.
+ *
+ *   Category 1: Control functions (zts_start, zts_join, zts_get_peer_status, etc). Errors
+ *                returned by these functions can be any of the following:
+ *
+ *      [ 0] ZTS_ERR_OK          - No error.
+ *      [-1] ZTS_ERR             - Error (see zts_errno for more information).
+ *      [-2] ZTS_ERR_INVALID_ARG - An argument provided is invalid.
+ *      [-3] ZTS_ERR_SERVICE     - ZT is not yet initialized. Try again.
+ *      [-4] ZTS_ERR_INVALID_OP  - Operation is not permitted (Doesn't make sense in this state).
+ *      [-5] ZTS_ERR_NO_RESULT   - Call succeeded but no result was available. Not always an error.
+ *      [-6] ZTS_ERR_GENERAL     - General internal failure. Consider filing a bug report.
+ *
+ *   Category 2: Sockets (zts_socket, zts_bind, zts_connect, zts_listen, etc).
+ *               Errors returned by these functions can be the same as the above. With
+ *               the added possibility of zts_errno being set. Much like standard
+ *               errno this will provide a more specific reason for an error's occurrence.
+ *               These error values are defined in: libzt/ext/lwip/src/include/lwip/errno.h
+ *               and closely map to standard Linux error values.
+ *
+ *
+ *   API COMPATIBILITY WITH HOST OS:
+ *
+ * - Since libzt re-implements a socket API probably very similar to your host OS's own
+ *   API it may be tempting to mix and match host OS structures and functions with those
+ *   of libzt. This may work on occasion, but you are tempting fate, so here are a few
+ *   guidelines:
+ *
+ *   If you are calling a zts_* function, use the appropriate ZTS_* constants:
+ *             
+ *          zts_socket(ZTS_AF_INET6, ZTS_SOCK_DGRAM, 0); (CORRECT)
+ *          zts_socket(AF_INET6, SOCK_DGRAM, 0);         (INCORRECT)
+ *
+ *   If you are calling a zts_* function, use the appropriate zts_* structure:
+ *
+ *          struct zts_sockaddr_in in4;  <------ Note the zts_* prefix
+ *             ...
+ *          zts_bind(fd, (struct sockaddr *)&in4, sizeof(struct zts_sockaddr_in)) < 0)
+ *
+ *   If you are calling a host OS function, use your host OS's constants (and structures!):
+ *       
+ *          inet_ntop(AF_INET6, &(in6->sin6_addr), ...);     (CORRECT)
+ *          inet_ntop(ZTS_AF_INET6, &(in6->sin6_addr), ...); (INCORRECT)
+ *
+ *   If you are calling a host OS function but passing a zts_* structure, this can
+ *       work sometimes but you should take care to pass the correct host OS constants:
+ *
+ *          struct zts_sockaddr_in6 in6;
+ *             ...
+ *          inet_ntop(AF_INET6, &(in6->sin6_addr), dstStr, INET6_ADDRSTRLEN);
+ */
+
+void delay_ms(long ms) { usleep(ms*1000); }
+
+bool nodeReady = false;
+bool networkReady = false;
+
+// Example callbacks
+void myZeroTierEventCallback(struct zts_callback_msg *msg)
+{
+	// Node events
+	if (msg->eventCode == ZTS_EVENT_NODE_ONLINE) {
+		NSLog(@"ZTS_EVENT_NODE_ONLINE --- This node's ID is %llx\n", msg->node->address);
+		nodeReady = true;
+	}
+	if (msg->eventCode == ZTS_EVENT_NODE_OFFLINE) {
+		NSLog(@"ZTS_EVENT_NODE_OFFLINE --- Check your physical Internet connection, router, firewall, etc. What ports are you blocking?\n");
+		nodeReady = false;
+	}
+	// Virtual network events
+	if (msg->eventCode == ZTS_EVENT_NETWORK_NOT_FOUND) {
+		NSLog(@"ZTS_EVENT_NETWORK_NOT_FOUND --- Are you sure %llx is a valid network?\n",
+			msg->network->nwid);
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_REQUESTING_CONFIG) {
+		NSLog(@"ZTS_EVENT_NETWORK_REQUESTING_CONFIG --- Requesting config for network %llx, please wait a few seconds...\n", msg->network->nwid);
+	} 
+	if (msg->eventCode == ZTS_EVENT_NETWORK_ACCESS_DENIED) {
+		NSLog(@"ZTS_EVENT_NETWORK_ACCESS_DENIED --- Access to virtual network %llx has been denied. Did you authorize the node yet?\n",
+			msg->network->nwid);
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_READY_IP6) {
+		NSLog(@"ZTS_EVENT_NETWORK_READY_IP6 --- Network config received. IPv6 traffic can now be sent over network %llx\n",
+			msg->network->nwid);
+		networkReady = true;
+	}
+	if (msg->eventCode == ZTS_EVENT_NETWORK_DOWN) {
+		NSLog(@"ZTS_EVENT_NETWORK_DOWN --- %llx\n", msg->network->nwid);
+	}
+	// Network stack events
+	if (msg->eventCode == ZTS_EVENT_NETIF_UP) {
+		NSLog(@"ZTS_EVENT_NETIF_UP --- network=%llx, mac=%llx, mtu=%d\n", 
+			msg->netif->nwid,
+			msg->netif->mac,
+			msg->netif->mtu);
+		networkReady = true;
+	}
+	if (msg->eventCode == ZTS_EVENT_NETIF_DOWN) {
+		NSLog(@"ZTS_EVENT_NETIF_DOWN --- network=%llx, mac=%llx\n", 
+			msg->netif->nwid,
+			msg->netif->mac);
+		
+		networkReady = true;
+	}
+	// Address events
+	if (msg->eventCode == ZTS_EVENT_ADDR_ADDED_IP6) {
+		char ipstr[INET6_ADDRSTRLEN];
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg->addr->addr);
+		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		NSLog(@"ZTS_EVENT_ADDR_NEW_IP6 --- Join %llx and ping me at %s\n", 
+			msg->addr->nwid, ipstr);
+	}
+	// Peer events
+	// If you don't recognize the peer ID, don't panic, this is most likely one of our root servers
+	if (msg->eventCode == ZTS_EVENT_PEER_P2P) {
+		NSLog(@"ZTS_EVENT_PEER_P2P --- There is now a direct path to peer %llx\n",
+			msg->peer->address);
+	}
+	if (msg->eventCode == ZTS_EVENT_PEER_RELAY) {
+		NSLog(@"ZTS_EVENT_PEER_RELAY --- No direct path to peer %llx\n",
+			msg->peer->address);
+	}
+}
+
+/*
+
+Ad-hoc Network:
+
+ffSSSSEEEE000000
+| |   |   |
+| |   |   Reserved for future use, must be 0
+| |   End of port range (hex)
+| Start of port range (hex)
+Reserved ZeroTier address prefix indicating a controller-less network.
+
+Ad-hoc networks are public (no access control) networks that have no network controller. Instead
+their configuration and other credentials are generated locally. Ad-hoc networks permit only IPv6
+UDP and TCP unicast traffic (no multicast or broadcast) using 6plane format NDP-emulated IPv6
+addresses. In addition an ad-hoc network ID encodes an IP port range. UDP packets and TCP SYN
+(connection open) packets are only allowed to destination ports within the encoded range.
+
+For example ff00160016000000 is an ad-hoc network allowing only SSH, while ff0000ffff000000 is an
+ad-hoc network allowing any UDP or TCP port.
+
+Keep in mind that these networks are public and anyone in the entire world can join them. Care must
+be taken to avoid exposing vulnerable services or sharing unwanted files or other resources.
+
+*/
+
+int main(int argc, char **argv) 
+{
+	if (argc != 5) {
+		NSLog(@"\nlibzt example\n");
+		NSLog(@"adhoc <config_file_path> <adhocStartPort> <adhocEndPort> <ztServicePort>\n");
+		exit(0);
+	}
+	int adhocStartPort = atoi(argv[2]); // Start of port range your application will use
+	int adhocEndPort = atoi(argv[3]); // End of port range your application will use
+	int ztServicePort = atoi(argv[4]); // Port ZT uses to send encrypted UDP packets to peers (try something like 9994)
+
+	uint64_t adhoc_nwid = zts_generate_adhoc_nwid_from_range(adhocStartPort, adhocEndPort);
+	int err = ZTS_ERR_OK;
+
+	zts_set_network_caching(false);
+
+	if((err = zts_start(argv[1], &myZeroTierEventCallback, ztServicePort)) != ZTS_ERR_OK) {
+		NSLog(@"Unable to start service, error = %d. Exiting.\n", err);
+		exit(1);
+	}
+	NSLog(@"Waiting for node to come online...\n");
+	while (!nodeReady) { delay_ms(50); }
+	NSLog(@"This node's identity is stored in %s\n", argv[1]);
+
+	if((err = zts_join(adhoc_nwid)) != ZTS_ERR_OK) {
+		NSLog(@"Unable to join network, error = %d. Exiting.\n", err);
+		exit(1);
+	}
+	NSLog(@"Joining network %llx\n", adhoc_nwid);
+	while (!networkReady) { delay_ms(50); }
+
+	// Idle and just show callback events, stack statistics, etc
+
+	NSLog(@"Node will now idle...\n");
+	while (true) { delay_ms(1000); }
+
+	// Shut down service and stack threads
+
+	zts_stop();
+	return 0;
+}

examples/swift/main.swift

@@ -0,0 +1,143 @@
+/**
+ * libzt Swift example
+ *
+ * swiftc -lc++ -import-objc-header ../../include/ZeroTierSockets.h -L. -lzt main.swift -o main;
+ * ./main
+ */
+
+import Swift
+import Foundation
+
+var nodeReady:Bool = false
+var networkReady:Bool = false
+
+let myZeroTierEventCallback : @convention(c) (UnsafeMutableRawPointer?) -> Void =
+{
+	(msgPtr) -> Void in
+	let msg = msgPtr?.bindMemory(to: zts_callback_msg.self, capacity: 1)
+
+	var eventCode = msg!.pointee.eventCode
+
+	let node = msg?.pointee.node;
+	let network = msg?.pointee.network;
+
+    switch Int32(eventCode)
+    {
+	case ZTS_EVENT_NODE_ONLINE:
+		let nodeId:UInt64 = node!.pointee.address
+		print(String(format: "ZTS_EVENT_NODE_ONLINE (%llx)", nodeId))
+		nodeReady = true;
+
+	case ZTS_EVENT_NODE_OFFLINE:
+		print("ZTS_EVENT_NODE_OFFLINE\n")
+		nodeReady = false;
+
+	case ZTS_EVENT_NODE_NORMAL_TERMINATION:
+		print("ZTS_EVENT_NODE_NORMAL_TERMINATION\n")
+
+	case ZTS_EVENT_NETWORK_NOT_FOUND:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_NOT_FOUND (%llx)", networkId))
+
+	case ZTS_EVENT_NETWORK_REQUESTING_CONFIG:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_REQUESTING_CONFIG (%llx)", networkId))
+
+	case ZTS_EVENT_NETWORK_ACCESS_DENIED:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_ACCESS_DENIED (%llx)", networkId))
+
+	case ZTS_EVENT_NETWORK_READY_IP4:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_READY_IP4 (%llx)", networkId))
+		networkReady = true;
+
+	case ZTS_EVENT_NETWORK_READY_IP6:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_READY_IP6 (%llx)", networkId))
+		networkReady = true;
+
+	case ZTS_EVENT_NETWORK_DOWN:
+		let networkId:UInt64 = network!.pointee.nwid
+		print(String(format: "ZTS_EVENT_NETWORK_DOWN (%llx)", networkId))
+
+/*
+	// Network stack events
+	case ZTS_EVENT_NETIF_UP:
+		print("ZTS_EVENT_NETIF_UP --- network=%llx, mac=%llx, mtu=%d\n", 
+			msg.netif->nwid,
+			msg.netif->mac,
+			msg.netif->mtu)
+		//networkReady = true;
+
+	case ZTS_EVENT_NETIF_DOWN:
+		print("ZTS_EVENT_NETIF_DOWN --- network=%llx, mac=%llx\n", 
+			msg.netif->nwid,
+			msg.netif->mac)
+		//networkReady = true;
+
+	// Address events
+	case ZTS_EVENT_ADDR_ADDED_IP4:
+		print("ZTS_EVENT_ADDR_ADDED_IP4")
+	/*
+		char ipstr[INET_ADDRSTRLEN];
+		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg.addr->addr);
+		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		print("ZTS_EVENT_ADDR_NEW_IP4 --- This node's virtual address on network %llx is %s\n", 
+			msg.addr->nwid, ipstr)
+			*/
+
+	case ZTS_EVENT_ADDR_ADDED_IP6:
+		print("ZTS_EVENT_ADDR_ADDED_IP6")
+		/*
+		char ipstr[INET6_ADDRSTRLEN];
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg.addr->addr);
+		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		print("ZTS_EVENT_ADDR_NEW_IP6 --- This node's virtual address on network %llx is %s\n", 
+			msg.addr->nwid, ipstr)
+			*/
+
+	case ZTS_EVENT_ADDR_REMOVED_IP4:
+		print("ZTS_EVENT_ADDR_REMOVED_IP4")
+		/*
+		char ipstr[INET_ADDRSTRLEN];
+		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)&(msg.addr->addr);
+		inet_ntop(AF_INET, &(in4->sin_addr), ipstr, INET_ADDRSTRLEN);
+		print("ZTS_EVENT_ADDR_REMOVED_IP4 --- The virtual address %s for this node on network %llx has been removed.\n", 
+			ipstr, msg.addr->nwid)
+			*/
+
+	case ZTS_EVENT_ADDR_REMOVED_IP6:
+		print("ZTS_EVENT_ADDR_REMOVED_IP6")
+	/*
+		char ipstr[INET6_ADDRSTRLEN];
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)&(msg.addr->addr);
+		inet_ntop(AF_INET6, &(in6->sin6_addr), ipstr, INET6_ADDRSTRLEN);
+		print("ZTS_EVENT_ADDR_REMOVED_IP6 --- The virtual address %s for this node on network %llx has been removed.\n", 
+			ipstr, msg.addr->nwid)
+		*/
+	// Peer events
+	case ZTS_EVENT_PEER_P2P:
+		print("ZTS_EVENT_PEER_P2P --- node=%llx\n", msg.peer->address)
+	case ZTS_EVENT_PEER_RELAY:
+		print("ZTS_EVENT_PEER_RELAY --- node=%llx\n", msg.peer->address)
+
+
+*/
+	default:
+		print("UNKNOWN_EVENT")
+	}
+}
+
+func main()
+{
+	print("waiting for node to come online...")
+	zts_start("../../config_path_a", myZeroTierEventCallback, 0)
+	while(!nodeReady) {
+		sleep(1)
+	}
+	print("Joining network")
+
+}
+
+main()

include/README.md

@@ -0,0 +1,4 @@
+ZeroTier Socket API
+======
+
+This is the externally facing plain C API. It provides a platform-agnostic ZeroTier-based socket interface.

include/Xcode-Bridging-Header.h

@@ -1,69 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-/**
- * @file
- *
- * ZeroTier socket API
- */
-
-#ifndef LIBZT_BRIDGING_HEADER_H
-#define LIBZT_BRIDGING_HEADER_H
-
-#include <sys/socket.h>
-#include "ZeroTier.h"
-
-//////////////////////////////////////////////////////////////////////////////
-// Service Controls                                                         //
-//////////////////////////////////////////////////////////////////////////////
-
-int zts_start(const char *path, void *callbackFunc, int port);
-void zts_stop();
-int zts_join(uint64_t nwid);
-int zts_leave(uint64_t nwid);
-uint64_t zts_get_node_id();
-uint64_t zts_get_node_status();
-int get_peer_status(uint64_t peerId);
-
-//////////////////////////////////////////////////////////////////////////////
-// Socket API                                                               //
-//////////////////////////////////////////////////////////////////////////////
-
-int zts_socket(int socket_family, int socket_type, int protocol);
-int zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen);
-int zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen);
-int zts_listen(int fd, int backlog);
-int zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen);
-int zts_setsockopt(int fd, int level, int optname, const void *optval, socklen_t optlen);
-int zts_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
-int zts_read(int fd, void *buf, size_t len);
-int zts_write(int fd, const void *buf, size_t len);
-ssize_t zts_send(int fd, const void *buf, size_t len, int flags);
-ssize_t zts_sendto(int fd, const void *buf, size_t len, int flags, const struct sockaddr *addr, socklen_t addrlen);
-ssize_t zts_sendmsg(int fd, const struct msghdr *msg, int flags);
-ssize_t zts_recv(int fd, void *buf, size_t len, int flags);
-ssize_t zts_recvfrom(int fd, void *buf, size_t len, int flags, struct sockaddr *addr, socklen_t *addrlen);
-ssize_t zts_recvmsg(int fd, struct msghdr *msg,int flags);
-int zts_shutdown(int fd, int how);
-int zts_close(int fd);
-int zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen);
-int zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen);
-int zts_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout);
-int zts_fcntl(int fd, int cmd, int flags);
-int zts_ioctl(int fd, unsigned long request, void *argp);
-
-#endif // _H
-
-
-
-

include/ZeroTierConstants.h

@@ -1,251 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-/**
- * @file
- *
- * Common constants used throughout the SDK
- */
-
-#ifndef ZEROTIER_CONSTANTS_H
-#define ZEROTIER_CONSTANTS_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// Custom errno to prevent conflicts with platform's own errno
-extern int zts_errno;
-
-#ifdef __cplusplus
-}
-#endif
-
-//////////////////////////////////////////////////////////////////////////////
-// Control API error codes                                                  //
-//////////////////////////////////////////////////////////////////////////////
-
-// No error.
-#define ZTS_ERR_OK                          0
-// Error (see zts_errno for more info)
-#define ZTS_ERR                            -1
-// A argument provided is invalid (e.g. out of range, NULL, etc)
-#define ZTS_ERR_INVALID_ARG                -2
-// The service isn't initialized or is currently unavailable. Try again.
-#define ZTS_ERR_SERVICE                    -3
-// This API operation is not permitted or doesn't make sense at this time.
-#define ZTS_ERR_INVALID_OP                 -4
-// The call succeeded, but no object or relevant result was available.
-#define ZTS_ERR_NO_RESULT                  -5
-// General internal failure. Consider filing a bug report.
-#define ZTS_ERR_GENERAL                    -6
-
-/**
- * The system port upon which ZT traffic is sent and received
- */
-#define ZTS_DEFAULT_PORT 9994
-
-//////////////////////////////////////////////////////////////////////////////
-// Control API event codes                                                  //
-//////////////////////////////////////////////////////////////////////////////
-
-#define ZTS_EVENT_NONE                     -1
-#define ZTS_EVENT_NODE_UP                   0
-// Standard node events
-#define ZTS_EVENT_NODE_OFFLINE              1
-#define ZTS_EVENT_NODE_ONLINE               2
-#define ZTS_EVENT_NODE_DOWN                 3
-#define ZTS_EVENT_NODE_IDENTITY_COLLISION   4
-#define ZTS_EVENT_NODE_UNRECOVERABLE_ERROR  16
-#define ZTS_EVENT_NODE_NORMAL_TERMINATION   17
-// Network events
-#define ZTS_EVENT_NETWORK_NOT_FOUND         32
-#define ZTS_EVENT_NETWORK_CLIENT_TOO_OLD    33
-#define ZTS_EVENT_NETWORK_REQUESTING_CONFIG 34
-#define ZTS_EVENT_NETWORK_OK                35
-#define ZTS_EVENT_NETWORK_ACCESS_DENIED     36
-#define ZTS_EVENT_NETWORK_READY_IP4         37
-#define ZTS_EVENT_NETWORK_READY_IP6         38
-#define ZTS_EVENT_NETWORK_READY_IP4_IP6     39
-#define ZTS_EVENT_NETWORK_DOWN              40
-// Network Stack events
-#define ZTS_EVENT_STACK_UP                  48
-#define ZTS_EVENT_STACK_DOWN                49
-// lwIP netif events
-#define ZTS_EVENT_NETIF_UP                  64
-#define ZTS_EVENT_NETIF_DOWN                65
-#define ZTS_EVENT_NETIF_REMOVED             66
-#define ZTS_EVENT_NETIF_LINK_UP             67
-#define ZTS_EVENT_NETIF_LINK_DOWN           68
-// Peer events
-#define ZTS_EVENT_PEER_P2P                  96
-#define ZTS_EVENT_PEER_RELAY                97
-#define ZTS_EVENT_PEER_UNREACHABLE          98
-// Path events
-#define ZTS_EVENT_PATH_DISCOVERED           112
-#define ZTS_EVENT_PATH_ALIVE                113
-#define ZTS_EVENT_PATH_DEAD                 114
-// Route events
-#define ZTS_EVENT_ROUTE_ADDED               128
-#define ZTS_EVENT_ROUTE_REMOVED             129
-// Address events
-#define ZTS_EVENT_ADDR_ADDED_IP4            144
-#define ZTS_EVENT_ADDR_REMOVED_IP4          145
-#define ZTS_EVENT_ADDR_ADDED_IP6            146
-#define ZTS_EVENT_ADDR_REMOVED_IP6          147
-
-// Macros for legacy behaviour
-#define NODE_EVENT_TYPE(code) code >= ZTS_EVENT_NODE_UP && code <= ZTS_EVENT_NODE_NORMAL_TERMINATION
-#define NETWORK_EVENT_TYPE(code) code >= ZTS_EVENT_NETWORK_NOT_FOUND && code <= ZTS_EVENT_NETWORK_DOWN
-#define STACK_EVENT_TYPE(code) code >= ZTS_EVENT_STACK_UP && code <= ZTS_EVENT_STACK_DOWN
-#define NETIF_EVENT_TYPE(code) code >= ZTS_EVENT_NETIF_UP && code <= ZTS_EVENT_NETIF_LINK_DOWN
-#define PEER_EVENT_TYPE(code) code >= ZTS_EVENT_PEER_P2P && code <= ZTS_EVENT_PEER_UNREACHABLE
-#define PATH_EVENT_TYPE(code) code >= ZTS_EVENT_PATH_DISCOVERED && code <= ZTS_EVENT_PATH_DEAD
-#define ROUTE_EVENT_TYPE(code) code >= ZTS_EVENT_ROUTE_ADDED && code <= ZTS_EVENT_ROUTE_REMOVED
-#define ADDR_EVENT_TYPE(code) code >= ZTS_EVENT_ADDR_ADDED_IP4 && code <= ZTS_EVENT_ADDR_REMOVED_IP6
-
-//////////////////////////////////////////////////////////////////////////////
-// Common definitions and structures for interacting with the ZT socket API //
-// This is a subset of lwip/sockets.h, lwip/arch.h, and lwip/inet.h         //
-//                                                                          //
-// These re-definitions exist here so that the user application's usage     //
-// of the API is internally consistent with the underlying network stack.   //
-// They have an attached prefix so that they can co-exist with the native   //
-// platform's own definitions and structures.                               // 
-//////////////////////////////////////////////////////////////////////////////
-
-// Socket protocol types
-#define ZTS_SOCK_STREAM     0x0001
-#define ZTS_SOCK_DGRAM      0x0002
-#define ZTS_SOCK_RAW        0x0003
-// Socket family types
-#define ZTS_AF_UNSPEC       0x0000
-#define ZTS_AF_INET         0x0002
-#define ZTS_AF_INET6        0x000a
-#define ZTS_PF_INET         ZTS_AF_INET
-#define ZTS_PF_INET6        ZTS_AF_INET6
-#define ZTS_PF_UNSPEC       ZTS_AF_UNSPEC
-// Protocol command types
-#define ZTS_IPPROTO_IP      0x0000
-#define ZTS_IPPROTO_ICMP    0x0001
-#define ZTS_IPPROTO_TCP     0x0006
-#define ZTS_IPPROTO_UDP     0x0011
-#define ZTS_IPPROTO_IPV6    0x0029
-#define ZTS_IPPROTO_ICMPV6  0x003a
-#define ZTS_IPPROTO_UDPLITE 0x0088
-#define ZTS_IPPROTO_RAW     0x00ff
-// send() and recv() flags
-#define ZTS_MSG_PEEK        0x0001
-#define ZTS_MSG_WAITALL     0x0002 // NOT YET SUPPORTED
-#define ZTS_MSG_OOB         0x0004 // NOT YET SUPPORTED
-#define ZTS_MSG_DONTWAIT    0x0008
-#define ZTS_MSG_MORE        0x0010
-// fnctl() commands
-#define ZTS_F_GETFL         0x0003
-#define ZTS_F_SETFL         0x0004
-// fnctl() flags
-#define ZTS_O_NONBLOCK      0x0001
-#define ZTS_O_NDELAY        0x0001
-// Shutdown commands
-#define ZTS_SHUT_RD         0x0000
-#define ZTS_SHUT_WR         0x0001
-#define ZTS_SHUT_RDWR       0x0002
-// Socket level option number
-#define ZTS_SOL_SOCKET      0x0fff
-// Socket options
-#define ZTS_SO_DEBUG        0x0001 // NOT YET SUPPORTED
-#define ZTS_SO_ACCEPTCONN   0x0002
-#define ZTS_SO_REUSEADDR    0x0004
-#define ZTS_SO_KEEPALIVE    0x0008
-#define ZTS_SO_DONTROUTE    0x0010 // NOT YET SUPPORTED
-#define ZTS_SO_BROADCAST    0x0020
-#define ZTS_SO_USELOOPBACK  0x0040 // NOT YET SUPPORTED
-#define ZTS_SO_LINGER       0x0080
-#define ZTS_SO_DONTLINGER   ((int)(~ZTS_SO_LINGER))
-#define ZTS_SO_OOBINLINE    0x0100 // NOT YET SUPPORTED
-#define ZTS_SO_REUSEPORT    0x0200 // NOT YET SUPPORTED
-#define ZTS_SO_SNDBUF       0x1001 // NOT YET SUPPORTED
-#define ZTS_SO_RCVBUF       0x1002
-#define ZTS_SO_SNDLOWAT     0x1003 // NOT YET SUPPORTED
-#define ZTS_SO_RCVLOWAT     0x1004 // NOT YET SUPPORTED
-#define ZTS_SO_SNDTIMEO     0x1005
-#define ZTS_SO_RCVTIMEO     0x1006
-#define ZTS_SO_ERROR        0x1007
-#define ZTS_SO_TYPE         0x1008
-#define ZTS_SO_CONTIMEO     0x1009
-#define ZTS_SO_NO_CHECK     0x100a
-// IPPROTO_IP options
-#define ZTS_IP_TOS          0x0001
-#define ZTS_IP_TTL          0x0002
-// IPPROTO_TCP options
-#define ZTS_TCP_NODELAY     0x0001
-#define ZTS_TCP_KEEPALIVE   0x0002
-#define ZTS_TCP_KEEPIDLE    0x0003
-#define ZTS_TCP_KEEPINTVL   0x0004
-#define ZTS_TCP_KEEPCNT     0x0005
-// IPPROTO_IPV6 options
-#define ZTS_IPV6_CHECKSUM   0x0007  // RFC3542
-#define ZTS_IPV6_V6ONLY     0x001b  // RFC3493
-// Macro's for defining ioctl() command values
-#define ZTS_IOCPARM_MASK    0x7fU
-#define ZTS_IOC_VOID        0x20000000UL
-#define ZTS_IOC_OUT         0x40000000UL
-#define ZTS_IOC_IN          0x80000000UL
-#define ZTS_IOC_INOUT       (ZTS_IOC_IN   | ZTS_IOC_OUT)
-#define ZTS_IO(x,y)         (ZTS_IOC_VOID | ((x)<<8)|(y))
-#define ZTS_IOR(x,y,t)      (ZTS_IOC_OUT  | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
-#define ZTS_IOW(x,y,t)      (ZTS_IOC_IN   | (((long)sizeof(t) & ZTS_IOCPARM_MASK)<<16) | ((x)<<8) | (y))
-// ioctl() commands
-#define ZTS_FIONREAD        ZTS_IOR('f', 127, unsigned long)
-#define ZTS_FIONBIO         ZTS_IOW('f', 126, unsigned long)
-
-/* FD_SET used for lwip_select */
-
-#ifndef ZTS_FD_SET
-#undef  ZTS_FD_SETSIZE
-// Make FD_SETSIZE match NUM_SOCKETS in socket.c
-#define ZTS_FD_SETSIZE    MEMP_NUM_NETCONN
-#define ZTS_FDSETSAFESET(n, code) do { \
-  if (((n) - LWIP_SOCKET_OFFSET < MEMP_NUM_NETCONN) && (((int)(n) - LWIP_SOCKET_OFFSET) >= 0)) { \
-  code; }} while(0)
-#define ZTS_FDSETSAFEGET(n, code) (((n) - LWIP_SOCKET_OFFSET < MEMP_NUM_NETCONN) && (((int)(n) - LWIP_SOCKET_OFFSET) >= 0) ?\
-  (code) : 0)
-#define ZTS_FD_SET(n, p)  ZTS_FDSETSAFESET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] |=  (1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
-#define ZTS_FD_CLR(n, p)  ZTS_FDSETSAFESET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] &= ~(1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
-#define ZTS_FD_ISSET(n,p) ZTS_FDSETSAFEGET(n, (p)->fd_bits[((n)-LWIP_SOCKET_OFFSET)/8] &   (1 << (((n)-LWIP_SOCKET_OFFSET) & 7)))
-#define ZTS_FD_ZERO(p)    memset((void*)(p), 0, sizeof(*(p)))
-
-#elif LWIP_SOCKET_OFFSET
-#error LWIP_SOCKET_OFFSET does not work with external FD_SET!
-#elif ZTS_FD_SETSIZE < MEMP_NUM_NETCONN
-#error "external ZTS_FD_SETSIZE too small for number of sockets"
-#endif // FD_SET
-
-//////////////////////////////////////////////////////////////////////////////
-// Statistics                                                               //
-//////////////////////////////////////////////////////////////////////////////
-
-#define ZTS_STATS_PROTOCOL_LINK      0
-#define ZTS_STATS_PROTOCOL_ETHARP    1
-#define ZTS_STATS_PROTOCOL_IP        2
-#define ZTS_STATS_PROTOCOL_UDP       3
-#define ZTS_STATS_PROTOCOL_TCP       4
-#define ZTS_STATS_PROTOCOL_ICMP      5
-#define ZTS_STATS_PROTOCOL_IP_FRAG   6
-#define ZTS_STATS_PROTOCOL_IP6       7
-#define ZTS_STATS_PROTOCOL_ICMP6     8
-#define ZTS_STATS_PROTOCOL_IP6_FRAG  9
-
-//#if defined(_USING_LWIP_DEFINITIONS_)
-
-#endif // ZEROTIER_CONSTANTS_H

src/Controls.hpp

@@ -1,107 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-/**
- * @file
- *
- * Header for ZeroTier service controls
- */
-
-#ifndef LIBZT_CONTROLS_HPP
-#define LIBZT_CONTROLS_HPP
-
-#ifdef _WIN32
-#include <Windows.h>
-#endif
-
-namespace ZeroTier {
-
-//////////////////////////////////////////////////////////////////////////////
-// ZeroTier Internal Service Controls                                       //
-//////////////////////////////////////////////////////////////////////////////
-
-/**
- * Add a callback event message to the queue. This can be safely called
- * from other threads since a lock-free queue is used.
- *
- * @param eventCode The event ID for this event
- * @param msg Pointer to a structure of pointers to other message-relevant
- * data structures.
- */
-void postEvent(int eventCode, void *arg);
-
-/**
- * Add a callback event message to the queue. This can be safely called
- * from other threads since a lock-free queue is used. Note: For use in
- * situations when no additional information needs to be conveyed to the
- * user application.
- *
- * @param eventCode The event ID for this event
- */
-void postEvent(int eventCode);
-
-/**
- * Free whatever was allocated to contain the callback message
- *
- * @param msg Message to be freed
- */
-void freeEvent(struct zts_callback_msg *msg);
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * @brief Starts a ZeroTier service in the background
- *
- * @usage For internal use only.
- * @param
- * @return
- */
-#if defined(_WIN32)
-DWORD WINAPI _zts_run_service(LPVOID thread_id);
-#else
-void *_zts_run_service(void *thread_id);
-#endif
-
-/**
- * @brief [Should not be called from user application] This function must be surrounded by 
- * ZT service locks. It will determine if it is currently safe and allowed to operate on 
- * the service.
- * @usage Can be called at any time
- * @return 1 or 0
- */
-int _zts_can_perform_service_operation();
-
-/**
- * @brief [Should not be called from user application] Returns whether or not the node is 
- * online.
- * @usage Can be called at any time
- * @return 1 or 0
- */
-int _zts_node_online();
-
-/**
- * @brief [Should not be called from user application] Adjusts the delay multiplier for the
- * network stack driver thread.
- * @usage Can be called at any time
- */
-void _hibernate_if_needed();
-
-#ifdef __cplusplus
-}
-#endif
-
-} // namespace ZeroTier
-
-#endif // _H

src/Events.cpp

@@ -0,0 +1,245 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2024-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+/**
+ * @file
+ *
+ * Callback event processing logic
+ */
+
+#include "concurrentqueue.h"
+
+#ifdef SDK_JNI
+	#include <jni.h>
+#endif
+
+#include "Node.hpp"
+#include "OSUtils.hpp"
+
+#include "Debug.hpp"
+#include "Events.hpp"
+#include "ZeroTierSockets.h"
+#include "NodeService.hpp"
+
+#define NODE_EVENT_TYPE(code) code >= ZTS_EVENT_NODE_UP && code <= ZTS_EVENT_NODE_NORMAL_TERMINATION
+#define NETWORK_EVENT_TYPE(code) code >= ZTS_EVENT_NETWORK_NOT_FOUND && code <= ZTS_EVENT_NETWORK_DOWN
+#define STACK_EVENT_TYPE(code) code >= ZTS_EVENT_STACK_UP && code <= ZTS_EVENT_STACK_DOWN
+#define NETIF_EVENT_TYPE(code) code >= ZTS_EVENT_NETIF_UP && code <= ZTS_EVENT_NETIF_LINK_DOWN
+#define PEER_EVENT_TYPE(code) code >= ZTS_EVENT_PEER_DIRECT && code <= ZTS_EVENT_PEER_UNREACHABLE
+#define PATH_EVENT_TYPE(code) code >= ZTS_EVENT_PATH_DISCOVERED && code <= ZTS_EVENT_PATH_DEAD
+#define ROUTE_EVENT_TYPE(code) code >= ZTS_EVENT_ROUTE_ADDED && code <= ZTS_EVENT_ROUTE_REMOVED
+#define ADDR_EVENT_TYPE(code) code >= ZTS_EVENT_ADDR_ADDED_IP4 && code <= ZTS_EVENT_ADDR_REMOVED_IP6
+
+namespace ZeroTier {
+
+extern NodeService *service;
+
+// Global state variable shared between Socket, Control, Event and NodeService logic.
+uint8_t _serviceStateFlags;
+
+// Lock to guard access to callback function pointers.
+Mutex _callbackLock;
+
+void (*_userEventCallbackFunc)(void *);
+
+moodycamel::ConcurrentQueue<struct ::zts_callback_msg*> _callbackMsgQueue;
+
+void _enqueueEvent(int16_t eventCode, void *arg)
+{
+	struct ::zts_callback_msg *msg = new ::zts_callback_msg();
+
+	msg->node = NULL;
+	msg->network = NULL;
+	msg->netif = NULL;
+	msg->route = NULL;
+	msg->path = NULL;
+	msg->peer = NULL;
+	msg->addr = NULL;
+
+	msg->eventCode = eventCode;
+
+	if (NODE_EVENT_TYPE(eventCode)) {
+		msg->node = (struct zts_node_details*)arg;
+	} if (NETWORK_EVENT_TYPE(eventCode)) {
+		msg->network = (struct zts_network_details*)arg;
+	} if (NETIF_EVENT_TYPE(eventCode)) {
+		msg->netif = (struct zts_netif_details*)arg;
+	} if (ROUTE_EVENT_TYPE(eventCode)) {
+		msg->route = (struct zts_virtual_network_route*)arg;
+	} if (PATH_EVENT_TYPE(eventCode)) {
+		msg->path = (struct zts_physical_path*)arg;
+	} if (PEER_EVENT_TYPE(eventCode)) {
+		msg->peer = (struct zts_peer_details*)arg;
+	} if (ADDR_EVENT_TYPE(eventCode)) {
+		msg->addr = (struct zts_addr_details*)arg;
+	}
+    _callbackMsgQueue.enqueue(msg);
+}
+
+void _freeEvent(struct ::zts_callback_msg *msg)
+{
+	if (!msg) {
+		return;
+	}
+	if (msg->node) { delete msg->node; }
+	if (msg->network) { delete msg->network; }
+	if (msg->netif) { delete msg->netif; }
+	if (msg->route) { delete msg->route; }
+	if (msg->path) { delete msg->path; }
+	if (msg->peer) { delete msg->peer; }
+	if (msg->addr) { delete msg->addr; }
+}
+
+void _passDequeuedEventToUser(struct ::zts_callback_msg *msg)
+{
+#ifdef SDK_JNI
+	if(_userCallbackMethodRef) {
+		JNIEnv *env;
+#if defined(__ANDROID__)
+		jint rs = jvm->AttachCurrentThread(&env, NULL);
+#else
+		jint rs = jvm->AttachCurrentThread((void **)&env, NULL);
+#endif
+		assert (rs == JNI_OK);
+		uint64_t arg = 0;
+		uint64_t id = 0;
+		if (NODE_EVENT_TYPE(msg->eventCode)) {
+			id = msg->node ? msg->node->address : 0;
+		}
+		if (NETWORK_EVENT_TYPE(msg->eventCode)) {
+			id = msg->network ? msg->network->nwid : 0;
+		}
+		if (PEER_EVENT_TYPE(msg->eventCode)) {
+			id = msg->peer ? msg->peer->address : 0;
+		}
+		env->CallVoidMethod(objRef, _userCallbackMethodRef, id, msg->eventCode);
+		_freeEvent(msg);
+	}
+#else
+	if (_userEventCallbackFunc) {
+		_userEventCallbackFunc(msg);
+		_freeEvent(msg);
+	}
+#endif
+}
+
+bool _isCallbackRegistered()
+{
+	_callbackLock.lock();
+	bool retval = false;
+#ifdef SDK_JNI
+	retval = (jvm && objRef && _userCallbackMethodRef);
+#else
+	retval = _userEventCallbackFunc;
+#endif
+	_callbackLock.unlock();
+	return retval;
+}
+
+void _clearRegisteredCallback()
+{
+	_callbackLock.lock();
+#ifdef SDK_JNI
+	objRef = NULL;
+    _userCallbackMethodRef = NULL;
+#else
+	_userEventCallbackFunc = NULL;
+#endif
+	_callbackLock.unlock();
+}
+
+int _canPerformServiceOperation()
+{
+	return service
+		&& service->isRunning()
+		&& service->getNode()
+		&& service->getNode()->online()
+		&& !_getState(ZTS_STATE_FREE_CALLED);
+}
+
+#define RESET_FLAGS( )   _serviceStateFlags  =  0;
+#define   SET_FLAGS(f)   _serviceStateFlags |=  f; 
+#define   CLR_FLAGS(f)   _serviceStateFlags &= ~f; 
+#define   GET_FLAGS(f) ((_serviceStateFlags &   f) > 0)
+
+void _setState(uint8_t newFlags)
+{
+	if ((newFlags ^ _serviceStateFlags) & ZTS_STATE_NET_SERVICE_RUNNING) {
+		return; // No effect. Not allowed to set this flag manually
+	}
+	SET_FLAGS(newFlags);
+	if (   GET_FLAGS(ZTS_STATE_NODE_RUNNING)
+      &&   GET_FLAGS(ZTS_STATE_STACK_RUNNING)
+      && !(GET_FLAGS(ZTS_STATE_FREE_CALLED)))
+	{
+		SET_FLAGS(ZTS_STATE_NET_SERVICE_RUNNING);
+	}
+	else {
+		CLR_FLAGS(ZTS_STATE_NET_SERVICE_RUNNING);
+	}
+}
+
+void _clrState(uint8_t newFlags)
+{
+	if (newFlags & ZTS_STATE_NET_SERVICE_RUNNING) {
+		return; // No effect. Not allowed to set this flag manually
+	}
+	CLR_FLAGS(newFlags);
+	if (   GET_FLAGS(ZTS_STATE_NODE_RUNNING)
+      &&   GET_FLAGS(ZTS_STATE_STACK_RUNNING)
+      && !(GET_FLAGS(ZTS_STATE_FREE_CALLED)))
+	{
+		SET_FLAGS(ZTS_STATE_NET_SERVICE_RUNNING);
+	}
+	else {
+		CLR_FLAGS(ZTS_STATE_NET_SERVICE_RUNNING);
+	}
+}
+
+bool _getState(uint8_t testFlags)
+{
+	return testFlags & _serviceStateFlags;
+}
+
+#if defined(__WINDOWS__)
+DWORD WINAPI _runCallbacks(LPVOID thread_id)
+#else
+void *_runCallbacks(void *thread_id)
+#endif
+{
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_EVENT_CALLBACK_THREAD_NAME);
+#endif
+	while (_getState(ZTS_STATE_CALLBACKS_RUNNING) || _callbackMsgQueue.size_approx() > 0)
+    {
+        struct ::zts_callback_msg *msg;
+		size_t sz = _callbackMsgQueue.size_approx();
+		for (size_t j = 0; j < sz; j++) {
+			if (_callbackMsgQueue.try_dequeue(msg)) {
+				_callbackLock.lock();
+				_passDequeuedEventToUser(msg);
+				_callbackLock.unlock();
+				delete msg;
+			}
+		}
+        zts_delay_ms(ZTS_CALLBACK_PROCESSING_INTERVAL);
+    }
+#if SDK_JNI
+	JNIEnv *env;
+	jint rs = jvm->DetachCurrentThread();
+    pthread_exit(0);
+#endif
+	return NULL;
+}
+
+} // namespace ZeroTier

src/Events.hpp

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2024-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+/**
+ * @file
+ *
+ * Header for callback event processing logic
+ */
+
+#ifndef ZT_EVENTS_HPP
+#define ZT_EVENTS_HPP
+
+#include <string>
+
+#include "ZeroTierSockets.h"
+
+#ifdef SDK_JNI
+	#include <jni.h>
+#endif
+namespace ZeroTier {
+
+#define ZTS_STATE_NODE_RUNNING              0x01
+#define ZTS_STATE_STACK_RUNNING             0x02
+#define ZTS_STATE_NET_SERVICE_RUNNING       0x04
+#define ZTS_STATE_CALLBACKS_RUNNING         0x08
+#define ZTS_STATE_FREE_CALLED               0x10
+
+#ifdef SDK_JNI
+	// References to JNI objects and VM kept for future callbacks
+	extern JavaVM *jvm;
+	extern jobject objRef;
+	extern jmethodID _userCallbackMethodRef;
+#endif
+
+/**
+ * How often callback messages are assembled and/or sent
+ */
+#define ZTS_CALLBACK_PROCESSING_INTERVAL 25
+
+/**
+ * Enqueue an event to be sent to the user application
+ */
+void _enqueueEvent(int16_t eventCode, void *arg);
+
+/**
+ * Send callback message to user application
+ */
+void _passDequeuedEventToUser(struct ::zts_callback_msg *msg);
+
+/**
+ * Free memory occupied by callback structures
+ */
+void _freeEvent(struct ::zts_callback_msg *msg);
+
+/**
+ * Return whether a callback method has been set
+ */
+bool _isCallbackRegistered();
+
+/**
+ * Clear pointer reference to user-provided callback function
+ */
+void _clearRegisteredCallback();
+
+/**
+ * Return whether service operation can be performed at this time
+ */
+int _canPerformServiceOperation();
+
+/**
+ * Set internal state flags
+ */
+void _setState(uint8_t newFlags);
+
+/**
+ * Clear internal state flags
+ */
+void _clrState(uint8_t newFlags);
+
+/**
+ * Get internal state flags
+ */
+bool _getState(uint8_t testFlags);
+
+#ifdef __WINDOWS__
+DWORD WINAPI _runCallbacks(LPVOID thread_id);
+#else
+/**
+ * Event callback thread
+ */
+void *_runCallbacks(void *thread_id);
+#endif
+
+} // namespace ZeroTier
+
+#endif // _H

src/NodeService.cpp

@@ -11,89 +11,57 @@
  */
 /****/
 
-#include <stdio.h>
+/**
-#include <stdlib.h>
+ * @file
-#include <string.h>
+ *
+ * ZeroTier Node Service (a distant relative of OneService)
+ */
 
-#include <string>
-#include <map>
-#include <vector>
-#include <algorithm>
-#include <list>
 #include <thread>
-#include <mutex>
-#include <condition_variable>
 
-#include "version.h"
+#include "Debug.hpp"
-#include "ZeroTierOne.h"
+#include "Events.hpp"
+#include "NodeService.hpp"
+#include "ZeroTierSockets.h"
+#include "VirtualTap.hpp"
 
 #include "Constants.hpp"
-#include "Mutex.hpp"
 #include "Node.hpp"
 #include "Utils.hpp"
-#include "InetAddress.hpp"
 #include "MAC.hpp"
-#include "Identity.hpp"
-#include "World.hpp"
-#include "Salsa20.hpp"
-#include "Poly1305.hpp"
-#include "SHA512.hpp"
-
 #include "Phy.hpp"
 #include "Thread.hpp"
 #include "OSUtils.hpp"
 #include "PortMapper.hpp"
 #include "Binder.hpp"
 #include "ManagedRoute.hpp"
+#include "InetAddress.hpp"
 #include "BlockingQueue.hpp"
 
-#include "Service.hpp"
+#if defined(__WINDOWS__)
-#include "Debug.hpp"
+WSADATA wsaData;
-#include "concurrentqueue.h"
-
-#include "ZeroTier.h"
-#include "lwipDriver.hpp"
-
-#ifdef __WINDOWS__
 #include <WinSock2.h>
 #include <Windows.h>
 #include <ShlObj.h>
 #include <netioapi.h>
 #include <iphlpapi.h>
-//#include <unistd.h>
 #define stat _stat
-#else
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-#include <unistd.h>
-#include <ifaddrs.h>
 #endif
 
-#include "Controls.hpp"
+#ifdef SDK_JNI
+#include <jni.h>
+#endif
 
-// Use the virtual netcon endpoint instead of a tun/tap port driver
+// Custom errno-like reporting variable
-#include "VirtualTap.hpp"
+int zts_errno;
-namespace ZeroTier { typedef VirtualTap EthernetTap; }
-
-// Interface metric for ZeroTier taps -- this ensures that if we are on WiFi and also
-// bridged via ZeroTier to the same LAN traffic will (if the OS is sane) prefer WiFi.
-#define ZT_IF_METRIC 5000
-
-// How often to check for new multicast subscriptions on a tap device
-#define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000
-
-// How often to check for local interface addresses
-#define ZT_LOCAL_INTERFACE_CHECK_INTERVAL 60000
 
 namespace ZeroTier {
 
-extern void postEvent(uint64_t id, int eventCode);
+uint8_t allowNetworkCaching;
-extern bool _network_caching_enabled;
+uint8_t allowPeerCaching;
-extern bool _peer_caching_enabled;
+uint8_t allowLocalConf;
 
-namespace {
+typedef VirtualTap EthernetTap;
 
 static std::string _trimString(const std::string &s)
 {
@@ -114,7 +82,7 @@ static std::string _trimString(const std::string &s)
 	return s.substr(start,end - start);
 }
 
-class OneServiceImpl;
+class NodeServiceImpl;
 
 static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf);
 static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData);
@@ -126,7 +94,7 @@ static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t z
 static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result);
 static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
-struct OneServiceIncomingPacket
+struct NodeServiceIncomingPacket
 {
 	uint64_t now;
 	int64_t sock;
@@ -135,7 +103,7 @@ struct OneServiceIncomingPacket
 	uint8_t data[ZT_MAX_MTU];
 };
 
-class OneServiceImpl : public OneService
+class NodeServiceImpl : public NodeService
 {
 public:
 	// begin member variables --------------------------------------------------
@@ -145,7 +113,7 @@ public:
 	const std::string _networksPath;
 	const std::string _moonsPath;
 
-	Phy<OneServiceImpl *> _phy;
+	Phy<NodeServiceImpl *> _phy;
 	Node *_node;
 	bool _updateAutoApply;
 	unsigned int _multipathMode = 0;
@@ -159,8 +127,8 @@ public:
 
 	//
 	unsigned long _incomingPacketConcurrency;
-	std::vector<OneServiceIncomingPacket *> _incomingPacketMemoryPool;
+	std::vector<NodeServiceIncomingPacket *> _incomingPacketMemoryPool;
-	BlockingQueue<OneServiceIncomingPacket *> _incomingPacketQueue;
+	BlockingQueue<NodeServiceIncomingPacket *> _incomingPacketQueue;
 	std::vector<std::thread> _incomingPacketThreads;
 	Mutex _incomingPacketMemoryPoolLock,_incomingPacketThreadsLock;
 
@@ -238,7 +206,7 @@ public:
 
 	// end member variables ----------------------------------------------------
 
-	OneServiceImpl(const char *hp,unsigned int port) :
+	NodeServiceImpl(const char *hp,unsigned int port) :
 		_homePath((hp) ? hp : ".")
 		,_phy(this,false,true)
 		,_node((Node *)0)
@@ -259,51 +227,16 @@ public:
 		_ports[1] = 0;
 		_ports[2] = 0;
 
-		/* Packet input concurrency is disabled intentially since it
+		allowNetworkCaching = true;
-		would force the user-space network stack to constantly re-order
+		allowPeerCaching = true;
-		frames, resulting in lower RX performance */
+		allowLocalConf = false;
-
+#ifdef __WINDOWS__
-		/*
+		// Initialize WinSock. Used in Phy for loopback pipe
-		_incomingPacketConcurrency = 1;
+		WSAStartup(MAKEWORD(2, 2), &wsaData);
-		// std::max((unsigned long)1,std::min((unsigned long)16,(unsigned long)std::thread::hardware_concurrency()));
+#endif
-		char *envPool = std::getenv("INCOMING_PACKET_CONCURRENCY");
-		if (envPool != NULL) {
-			int tmp = atoi(envPool);
-			if (tmp > 0) {
-				_incomingPacketConcurrency = tmp;
-			}
-		}
-		for(long t=0;t<_incomingPacketConcurrency;++t) {
-			_incomingPacketThreads.push_back(std::thread([this]() {
-				OneServiceIncomingPacket *pkt = nullptr;
-				for(;;) {
-					if (!_incomingPacketQueue.get(pkt))
-						break;
-					if (!pkt)
-						break;
-					if (!_run)
-						break;
-
-					const ZT_ResultCode rc = _node->processWirePacket(nullptr,pkt->now,pkt->sock,&(pkt->from),pkt->data,pkt->size,&_nextBackgroundTaskDeadline);
-					{
-						Mutex::Lock l(_incomingPacketMemoryPoolLock);
-						_incomingPacketMemoryPool.push_back(pkt);
-					}
-					if (ZT_ResultCode_isFatal(rc)) {
-						char tmp[256];
-						OSUtils::ztsnprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc);
-						Mutex::Lock _l(_termReason_m);
-						_termReason = ONE_UNRECOVERABLE_ERROR;
-						_fatalErrorMessage = tmp;
-						this->terminate();
-						break;
-					}
-				}
-			}));
-		}*/
 	}
 
-	virtual ~OneServiceImpl()
+	virtual ~NodeServiceImpl()
 	{
 		_incomingPacketQueue.stop();
 		_incomingPacketThreadsLock.lock();
@@ -425,7 +358,7 @@ public:
 			}
 #endif
 			// Join existing networks in networks.d
-			if (_network_caching_enabled) {
+			if (allowNetworkCaching) {
 				std::vector<std::string> networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str()));
 				for(std::vector<std::string>::iterator f(networksDotD.begin());f!=networksDotD.end();++f) {
 					std::size_t dot = f->find_last_of('.');
@@ -769,7 +702,7 @@ public:
 					*nuptr = (void *)0;
 					delete n.tap;
 					_nets.erase(nwid);
-					if (_network_caching_enabled) {
+					if (allowNetworkCaching) {
 						if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) {
 							char nlcpath[256];
 							OSUtils::ztsnprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid);
@@ -787,17 +720,25 @@ public:
 	inline void nodeEventCallback(enum ZT_Event event,const void *metaData)
 	{
 		// Feed node events into lock-free queue for later dequeuing by the callback thread
-		if (event <= ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION) {
+		switch(event) {
-			if (event == ZTS_EVENT_NODE_ONLINE) {
+			case ZT_EVENT_UP: {
+				_enqueueEvent(ZTS_EVENT_NODE_UP, NULL);
+			}	break;
+			case ZT_EVENT_ONLINE: {
 				struct zts_node_details *nd = new zts_node_details;
 				nd->address = _node->address();
-				postEvent(event, (void*)nd);
+				_enqueueEvent(ZTS_EVENT_NODE_ONLINE, (void*)nd);
-			}
+			}	break;
-			else {
+			case ZT_EVENT_OFFLINE: {
-				postEvent(event, (void*)0);
+				struct zts_node_details *nd = new zts_node_details;
-			}
+				nd->address = _node->address();
-		}
+				_enqueueEvent(ZTS_EVENT_NODE_OFFLINE, (void*)nd);
-		switch(event) {
+			}	break;
+			case ZT_EVENT_DOWN: {
+				struct zts_node_details *nd = new zts_node_details;
+				nd->address = _node->address();
+				_enqueueEvent(ZTS_EVENT_NODE_DOWN, (void*)nd);
+			}	break;
 			case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION: {
 				Mutex::Lock _l(_termReason_m);
 				_termReason = ONE_IDENTITY_COLLISION;
@@ -828,7 +769,7 @@ public:
 	{
 		// Force the ordering of callback messages, these messages are
 		// only useful if the node and stack are both up and running
-		if (!_node->online() || !lwip_is_up()) {
+		if (!_node->online() || !_lwip_is_up()) {
 			return;
 		}
 		// Generate messages to be dequeued by the callback message thread
@@ -842,26 +783,26 @@ public:
 			}
 			switch (mostRecentStatus) {
 				case ZT_NETWORK_STATUS_NOT_FOUND:
-					postEvent(ZTS_EVENT_NETWORK_NOT_FOUND, (void*)prepare_network_details_msg(nwid));
+					_enqueueEvent(ZTS_EVENT_NETWORK_NOT_FOUND, (void*)prepare_network_details_msg(nwid));
 					break;
 				case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:
-					postEvent(ZTS_EVENT_NETWORK_CLIENT_TOO_OLD, (void*)prepare_network_details_msg(nwid));
+					_enqueueEvent(ZTS_EVENT_NETWORK_CLIENT_TOO_OLD, (void*)prepare_network_details_msg(nwid));
 					break;
 				case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION:
-					postEvent(ZTS_EVENT_NETWORK_REQUESTING_CONFIG, (void*)prepare_network_details_msg(nwid));
+					_enqueueEvent(ZTS_EVENT_NETWORK_REQ_CONFIG, (void*)prepare_network_details_msg(nwid));
 					break;
 				case ZT_NETWORK_STATUS_OK:
-					if (tap->hasIpv4Addr() && lwip_is_netif_up(tap->netif4)) {
+					if (tap->hasIpv4Addr() && _lwip_is_netif_up(tap->netif4)) {
-						postEvent(ZTS_EVENT_NETWORK_READY_IP4, (void*)prepare_network_details_msg(nwid));
+						_enqueueEvent(ZTS_EVENT_NETWORK_READY_IP4, (void*)prepare_network_details_msg(nwid));
 					}
-					if (tap->hasIpv6Addr() && lwip_is_netif_up(tap->netif6)) {
+					if (tap->hasIpv6Addr() && _lwip_is_netif_up(tap->netif6)) {
-						postEvent(ZTS_EVENT_NETWORK_READY_IP6, (void*)prepare_network_details_msg(nwid));
+						_enqueueEvent(ZTS_EVENT_NETWORK_READY_IP6, (void*)prepare_network_details_msg(nwid));
 					}
 					// In addition to the READY messages, send one OK message
-					postEvent(ZTS_EVENT_NETWORK_OK, (void*)prepare_network_details_msg(nwid));
+					_enqueueEvent(ZTS_EVENT_NETWORK_OK, (void*)prepare_network_details_msg(nwid));
 					break;
 				case ZT_NETWORK_STATUS_ACCESS_DENIED:
-					postEvent(ZTS_EVENT_NETWORK_ACCESS_DENIED, (void*)prepare_network_details_msg(nwid));
+					_enqueueEvent(ZTS_EVENT_NETWORK_ACCESS_DENIED, (void*)prepare_network_details_msg(nwid));
 					break;
 				default:
 					break;
@@ -879,12 +820,12 @@ public:
 					if (pl->peers[i].pathCount > 0) {
 						pd = new zts_peer_details;
 						memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details));
-						postEvent(ZTS_EVENT_PEER_P2P, (void*)pd);
+						_enqueueEvent(ZTS_EVENT_PEER_DIRECT, (void*)pd);
 					}
 					if (pl->peers[i].pathCount == 0) {
 						pd = new zts_peer_details;
 						memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details));
-						postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
+						_enqueueEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
 					}
 				}
 				// Previously known peer, update status
@@ -892,12 +833,12 @@ public:
 					if ((peerCache[pl->peers[i].address] == false) && pl->peers[i].pathCount > 0) {
 						pd = new zts_peer_details;
 						memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details));
-						postEvent(ZTS_EVENT_PEER_P2P, (void*)pd);
+						_enqueueEvent(ZTS_EVENT_PEER_DIRECT, (void*)pd);
 					}
 					if ((peerCache[pl->peers[i].address] == true) && pl->peers[i].pathCount == 0) {
 						pd = new zts_peer_details;
 						memcpy(pd, &(pl->peers[i]), sizeof(struct zts_peer_details));
-						postEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
+						_enqueueEvent(ZTS_EVENT_PEER_RELAY, (void*)pd);
 					}
 				}
 				// Update our cache with most recently observed path count
@@ -938,7 +879,7 @@ public:
 		if (pl) {
 			for(unsigned long i=0;i<pl->peerCount;++i) {
 				if (pl->peers[i].address == id) {
-					status = pl->peers[i].pathCount > 0 ? ZTS_EVENT_PEER_P2P : ZTS_EVENT_PEER_RELAY;
+					status = pl->peers[i].pathCount > 0 ? ZTS_EVENT_PEER_DIRECT : ZTS_EVENT_PEER_RELAY;
 					break;
 				}
 			}
@@ -967,7 +908,7 @@ public:
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str());
 				break;
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
-				if (_network_caching_enabled) {
+				if (allowNetworkCaching) {
 					OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "networks.d",_homePath.c_str());
 					OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.conf",dirname,(unsigned long long)id[0]);
 					secure = true;
@@ -977,7 +918,7 @@ public:
 				}
 				break;
 			case ZT_STATE_OBJECT_PEER:
-				if (_peer_caching_enabled) {
+				if (allowPeerCaching) {
 					OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "peers.d",_homePath.c_str());
 					OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.10llx.peer",dirname,(unsigned long long)id[0]);
 				}
@@ -1032,7 +973,7 @@ public:
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str());
 				break;
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
-				if (_network_caching_enabled) {
+				if (allowNetworkCaching) {
 					OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]);
 				}
 				else {
@@ -1040,7 +981,7 @@ public:
 				}
 				break;
 			case ZT_STATE_OBJECT_PEER:
-				if (_peer_caching_enabled) {
+				if (allowPeerCaching) {
 					OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "peers.d" ZT_PATH_SEPARATOR_S "%.10llx.peer",_homePath.c_str(),(unsigned long long)id[0]);
 				}
 				break;
@@ -1249,32 +1190,121 @@ public:
 };
 
 static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); }
+{ return reinterpret_cast<NodeServiceImpl *>(uptr)->nodeVirtualNetworkConfigFunction(nwid,nuptr,op,nwconf); }
 static void SnodeEventCallback(ZT_Node *node,void *uptr,void *tptr,enum ZT_Event event,const void *metaData)
-{ reinterpret_cast<OneServiceImpl *>(uptr)->nodeEventCallback(event,metaData); }
+{ reinterpret_cast<NodeServiceImpl *>(uptr)->nodeEventCallback(event,metaData); }
 static void SnodeStatePutFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],const void *data,int len)
-{ reinterpret_cast<OneServiceImpl *>(uptr)->nodeStatePutFunction(type,id,data,len); }
+{ reinterpret_cast<NodeServiceImpl *>(uptr)->nodeStatePutFunction(type,id,data,len); }
 static int SnodeStateGetFunction(ZT_Node *node,void *uptr,void *tptr,enum ZT_StateObjectType type,const uint64_t id[2],void *data,unsigned int maxlen)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeStateGetFunction(type,id,data,maxlen); }
+{ return reinterpret_cast<NodeServiceImpl *>(uptr)->nodeStateGetFunction(type,id,data,maxlen); }
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,void *tptr,int64_t localSocket,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localSocket,addr,data,len,ttl); }
+{ return reinterpret_cast<NodeServiceImpl *>(uptr)->nodeWirePacketSendFunction(localSocket,addr,data,len,ttl); }
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t nwid,void **nuptr,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
-{ reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); }
+{ reinterpret_cast<NodeServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,nuptr,sourceMac,destMac,etherType,vlanId,data,len); }
 static int SnodePathCheckFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int64_t localSocket,const struct sockaddr_storage *remoteAddr)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodePathCheckFunction(ztaddr,localSocket,remoteAddr); }
+{ return reinterpret_cast<NodeServiceImpl *>(uptr)->nodePathCheckFunction(ztaddr,localSocket,remoteAddr); }
 static int SnodePathLookupFunction(ZT_Node *node,void *uptr,void *tptr,uint64_t ztaddr,int family,struct sockaddr_storage *result)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodePathLookupFunction(ztaddr,family,result); }
+{ return reinterpret_cast<NodeServiceImpl *>(uptr)->nodePathLookupFunction(ztaddr,family,result); }
 static void StapFrameHandler(void *uptr,void *tptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
-{ reinterpret_cast<OneServiceImpl *>(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); }
+{ reinterpret_cast<NodeServiceImpl *>(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); }
 
-} // anonymous namespace
 
-std::string OneService::platformDefaultHomePath()
+std::string NodeService::platformDefaultHomePath()
 {
 	return OSUtils::platformDefaultHomePath();
 }
 
-OneService *OneService::newInstance(const char *hp,unsigned int port) { return new OneServiceImpl(hp,port); }
+NodeService *NodeService::newInstance(const char *hp,unsigned int port) { return new NodeServiceImpl(hp,port); }
-OneService::~OneService() {}
+NodeService::~NodeService() {}
+
+//////////////////////////////////////////////////////////////////////////////
+// Service                                                                  //
+//////////////////////////////////////////////////////////////////////////////
+
+NodeService *service;
+
+// Lock to guard access to ZeroTier core service
+Mutex serviceLock;
+
+// Starts a ZeroTier NodeService background thread
+#if defined(__WINDOWS__)
+DWORD WINAPI _runNodeService(LPVOID arg)
+#else
+void *_runNodeService(void *arg)
+#endif
+{	
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_SERVICE_THREAD_NAME);
+#endif
+	struct serviceParameters *params = (struct serviceParameters *)arg;
+	int err;
+	try {
+		std::vector<std::string> hpsp(OSUtils::split(params->path.c_str(), ZT_PATH_SEPARATOR_S,"",""));
+		std::string ptmp;
+		if (params->path[0] == ZT_PATH_SEPARATOR) {
+			ptmp.push_back(ZT_PATH_SEPARATOR);
+		}
+		for (std::vector<std::string>::iterator pi(hpsp.begin());pi!=hpsp.end();++pi) {
+			if (ptmp.length() > 0) {
+				ptmp.push_back(ZT_PATH_SEPARATOR);
+			}
+			ptmp.append(*pi);
+			if ((*pi != ".")&&(*pi != "..")) {
+				if (OSUtils::mkdir(ptmp) == false) {
+					DEBUG_ERROR("home path does not exist, and could not create");
+					err = true;
+					perror("error\n");
+				}
+			}
+		}
+		for(;;) {
+			serviceLock.lock();
+			service = NodeService::newInstance(params->path.c_str(),params->port);
+			service->_userProvidedPort = params->port;
+			service->_userProvidedPath = params->path;
+			serviceLock.unlock();
+			switch(service->run()) {
+				case NodeService::ONE_STILL_RUNNING:
+				case NodeService::ONE_NORMAL_TERMINATION:
+					_enqueueEvent(ZTS_EVENT_NODE_NORMAL_TERMINATION,NULL);
+					break;
+				case NodeService::ONE_UNRECOVERABLE_ERROR:
+					DEBUG_ERROR("fatal error: %s", service->fatalErrorMessage().c_str());
+					err = true;
+					_enqueueEvent(ZTS_EVENT_NODE_UNRECOVERABLE_ERROR,NULL);
+					break;
+				case NodeService::ONE_IDENTITY_COLLISION: {
+					err = true;
+					delete service;
+					service = (NodeService *)0;
+					std::string oldid;
+					OSUtils::readFile((params->path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str(),oldid);
+					if (oldid.length()) {
+						OSUtils::writeFile((params->path + ZT_PATH_SEPARATOR_S + "identity.secret.saved_after_collision").c_str(),oldid);
+						OSUtils::rm((params->path + ZT_PATH_SEPARATOR_S + "identity.secret").c_str());
+						OSUtils::rm((params->path + ZT_PATH_SEPARATOR_S + "identity.public").c_str());
+					}
+					_enqueueEvent(ZTS_EVENT_NODE_IDENTITY_COLLISION,NULL);
+				}	continue; // restart!
+			}
+			break; // terminate loop -- normally we don't keep restarting
+		}
+		serviceLock.lock();
+		_clrState(ZTS_STATE_NODE_RUNNING);
+		delete service;
+		service = (NodeService *)0;
+		serviceLock.unlock();
+		_enqueueEvent(ZTS_EVENT_NODE_DOWN,NULL);
+	} catch ( ... ) {
+		DEBUG_ERROR("unexpected exception starting ZeroTier instance");
+	}
+	delete params;
+	zts_delay_ms(ZTS_CALLBACK_PROCESSING_INTERVAL*2);
+	_clrState(ZTS_STATE_CALLBACKS_RUNNING);
+#ifndef __WINDOWS__
+	pthread_exit(0);
+#endif
+	return NULL;
+}
 
 } // namespace ZeroTier

src/NodeService.hpp

@@ -11,33 +11,49 @@
  */
 /****/
 
-#ifndef ZT_ONESERVICE_HPP
+/**
-#define ZT_ONESERVICE_HPP
+ * @file
+ *
+ * Header for ZeroTier Node Service (a distant relative of OneService)
+ */
+
+#ifndef ZT_NODE_SERVICE_HPP
+#define ZT_NODE_SERVICE_HPP
 
 #include <string>
 #include <vector>
 
-#ifdef SDK_JNI
+#include "Node.hpp"
-#include <jni.h>
+#include "InetAddress.hpp"
+#include "Mutex.hpp"
+#include "ZeroTierSockets.h"
+
+#define ZTS_SERVICE_THREAD_NAME           "ZTServiceThread"
+#define ZTS_EVENT_CALLBACK_THREAD_NAME    "ZTEventCallbackThread"
+// Interface metric for ZeroTier taps -- this ensures that if we are on WiFi and also
+// bridged via ZeroTier to the same LAN traffic will (if the OS is sane) prefer WiFi.
+#define ZT_IF_METRIC                      5000
+// How often to check for new multicast subscriptions on a tap device
+#define ZT_TAP_CHECK_MULTICAST_INTERVAL   5000
+// How often to check for local interface addresses
+#define ZT_LOCAL_INTERFACE_CHECK_INTERVAL 60000
+
+#ifdef __WINDOWS__
+#include <Windows.h>
 #endif
 
 namespace ZeroTier {
 
-class VirtualTap;
-// Use the virtual libzt endpoint instead of a tun/tap port driver
-namespace ZeroTier { typedef VirtualTap EthernetTap; }
-
-// Forward declaration so we can avoid dragging everything in
-struct InetAddress;
-class Node;
-
 /**
  * Local service for ZeroTier One as system VPN/NFV provider
  */
-class OneService
+class NodeService
 {
 public:
 
+	uint16_t _userProvidedPort;
+	std::string _userProvidedPath;
+
 	/**
 	 * Returned by node main if/when it terminates
 	 */
@@ -109,9 +125,9 @@ public:
 	 * @param hp Home path
 	 * @param port TCP and UDP port for packets and HTTP control (if 0, pick random port)
 	 */
-	static OneService *newInstance(const char *hp,unsigned int port);
+	static NodeService *newInstance(const char *hp,unsigned int port);
 
-	virtual ~OneService();
+	virtual ~NodeService();
 
 	/**
 	 * Execute the service main I/O loop until terminated
@@ -178,13 +194,28 @@ public:
 	inline bool isRunning() const { return (this->reasonForTermination() == ONE_STILL_RUNNING); }
 
 protected:
-	OneService() {}
+	NodeService() {}
 
 private:
-	OneService(const OneService &one) {}
+	NodeService(const NodeService &one) {}
-	inline OneService &operator=(const OneService &one) { return *this; }
+	inline NodeService &operator=(const NodeService &one) { return *this; }
 };
 
+struct serviceParameters
+{
+	int port;
+	std::string path;
+};
+
+#ifdef __WINDOWS__
+DWORD WINAPI _runNodeService(LPVOID arg);
+#else
+/**
+ * NodeService thread
+ */
+void *_runNodeService(void *arg);
+#endif
+
 } // namespace ZeroTier
 
 #endif

src/Options.h

@@ -1,80 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef LIBZT_OPTIONS_H
-#define LIBZT_OPTIONS_H
-
-//////////////////////////////////////////////////////////////////////////////
-// Callbacks                                                                //
-//////////////////////////////////////////////////////////////////////////////
-
-/**
- * The maximum number of un-processed callback messages
- */
-#define ZTS_CALLBACK_MSG_QUEUE_LEN 256
-
-//////////////////////////////////////////////////////////////////////////////
-// Timing                                                                   //
-//////////////////////////////////////////////////////////////////////////////
-
-/**
- * How often callback messages are assembled and/or sent
- */
-#define ZTS_CALLBACK_PROCESSING_INTERVAL 25
-
-/**
- * Polling interval (in ms) for fds wrapped in the Phy I/O loop
- */
-#define ZTS_TAP_THREAD_POLLING_INTERVAL 50
-
-#define ZTS_HOUSEKEEPING_INTERVAL 50
-
-/**
- * By how much thread I/O and callback loop delays are multiplied (unitless)
- */
-#define ZTS_HIBERNATION_MULTIPLIER 50
-
-//////////////////////////////////////////////////////////////////////////////
-// Threading                                                                //
-//////////////////////////////////////////////////////////////////////////////
-
-#define SERVICE_THREAD_NICENESS       0 // -10
-#define CALLBACK_THREAD_NICENESS      0 //  10
-#define LWIP_DRIVER_THREAD_NICENESS   0 //  10
-#define TCPIP_THREAD_NICENESS         0 // -10
-#define TAP_THREAD_NICENESS           0 //  10
-
-#define ZTS_SERVICE_THREAD_NAME "ZeroTierServiceThread"
-#define ZTS_EVENT_CALLBACK_THREAD_NAME "ZeroTierEventCallbackThread"
-#define ZTS_LWIP_DRIVER_THREAD_NAME "lwipDriver"
-
-//////////////////////////////////////////////////////////////////////////////
-// lwIP behaviour (tcpip driver)                                            //
-//////////////////////////////////////////////////////////////////////////////
-
-/**
- * How many frames are handled per call from core
- */
-#define LWIP_FRAMES_HANDLED_PER_CORE_CALL 16
-
-/**
- * How often the lwIP tcpip thread callback checks for incoming frames
- */
-#define LWIP_DRIVER_LOOP_INTERVAL 250
-
-/**
- * Number of packets that can be queued for ingress into the lwIP core
- */
-#define ZTS_LWIP_MAX_RX_QUEUE_LEN 1024
-
-#endif

src/Sockets.cpp

@@ -17,42 +17,41 @@
  * ZeroTier Socket API
  */
 
-#include <string.h>
-
 #include "lwip/sockets.h"
 #include "lwip/def.h"
+#include "lwip/inet.h"
+#include "lwip/stats.h"
 
-#include "ZeroTierConstants.h"
+#include "ZeroTierSockets.h"
-#include "Options.h"
+#include "Events.hpp"
-#include "Debug.hpp"
 
 #ifdef SDK_JNI
 	#include <jni.h>
 #endif
 
+extern int zts_errno;
+
 namespace ZeroTier {
 
-//////////////////////////////////////////////////////////////////////////////
+extern uint8_t _serviceStateFlags;
-// ZeroTier Socket API                                                      //
-//////////////////////////////////////////////////////////////////////////////
-
-extern bool _run_service;
-extern bool _run_lwip_tcpip;
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #ifdef SDK_JNI
-void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+void ss2zta(JNIEnv *env, struct zts_sockaddr_storage *ss, jobject addr);
-void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr);
+void zta2ss(JNIEnv *env, struct zts_sockaddr_storage *ss, jobject addr);
-void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set);
+void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, zts_fd_set *dest_fd_set);
-void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set);
+void fdset2ztfdset(JNIEnv *env, int nfds, zts_fd_set *src_fd_set, jobject dest_ztfd_set);
 #endif
 
-int zts_socket(int socket_family, int socket_type, int protocol)
+int zts_socket(const int socket_family, const int socket_type, const int protocol)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_socket(socket_family, socket_type, protocol);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_socket(socket_family, socket_type, protocol);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
@@ -63,53 +62,62 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_socket(
 }
 #endif
 
-int zts_connect(int fd, const struct sockaddr *addr, socklen_t addrlen)
+int zts_connect(int fd, const struct zts_sockaddr *addr, zts_socklen_t addrlen)
 {
 	if (!addr) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+	if (addrlen > (int)sizeof(struct zts_sockaddr_storage) || addrlen < (int)sizeof(struct zts_sockaddr_in)) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_connect(fd, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_connect(fd, (sockaddr*)addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_connect(
 	JNIEnv *env, jobject thisObj, jint fd, jobject addr)
 {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
-	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	socklen_t addrlen = ss.ss_family == ZTS_AF_INET ? sizeof(struct zts_sockaddr_in) : sizeof(struct zts_sockaddr_in6);
-	int retval = zts_connect(fd, (struct sockaddr *)&ss, addrlen);
+	int retval = zts_connect(fd, (struct zts_sockaddr *)&ss, addrlen);
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
-int zts_bind(int fd, const struct sockaddr *addr, socklen_t addrlen)
+int zts_bind(int fd, const struct zts_sockaddr *addr, zts_socklen_t addrlen)
 {
 	if (!addr) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+	if (addrlen > (int)sizeof(struct zts_sockaddr_storage) || addrlen < (int)sizeof(struct zts_sockaddr_in)) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_bind(fd, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_bind(fd, (sockaddr*)addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_bind(
 	JNIEnv *env, jobject thisObj, jint fd, jobject addr)
 {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
-	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	zts_socklen_t addrlen = ss.ss_family == ZTS_AF_INET ? sizeof(struct zts_sockaddr_in) : sizeof(struct zts_sockaddr_in6);
-	int retval = zts_bind(fd, (struct sockaddr*)&ss, addrlen);
+	int retval = zts_bind(fd, (struct zts_sockaddr*)&ss, addrlen);
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
 int zts_listen(int fd, int backlog)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_listen(fd, backlog);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_listen(fd, backlog);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(
@@ -120,26 +128,32 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_listen(
 }
 #endif
 
-int zts_accept(int fd, struct sockaddr *addr, socklen_t *addrlen)
+int zts_accept(int fd, struct zts_sockaddr *addr, zts_socklen_t *addrlen)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_accept(fd, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_accept(fd, (sockaddr*)addr, (socklen_t*)addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept(
 	JNIEnv *env, jobject thisObj, jint fd, jobject addr, jint port)
 {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
-	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	zts_socklen_t addrlen = sizeof(struct zts_sockaddr_storage);
-	int retval = zts_accept(fd, (struct sockaddr *)&ss, &addrlen);
+	int retval = zts_accept(fd, (zts_sockaddr*)&ss, &addrlen);
 	ss2zta(env, &ss, addr);
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
 #if defined(__linux__)
-int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)
+int zts_accept4(int fd, struct zts_sockaddr *addr, zts_socklen_t *addrlen, int flags)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP;
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return ZTS_ERR_SERVICE; // TODO
 }
 #endif
 #ifdef SDK_JNI
@@ -147,18 +161,21 @@ int zts_accept4(int fd, struct sockaddr *addr, socklen_t *addrlen, int flags)
  JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_accept4(
 	JNIEnv *env, jobject thisObj, jint fd, jobject addr, jint port, jint flags)
  {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
-	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	zts_socklen_t addrlen = sizeof(struct zts_sockaddr_storage);
-	int retval = zts_accept4(fd, (struct sockaddr *)&ss, &addrlen, flags);
+	int retval = zts_accept4(fd, (struct zts_sockaddr *)&ss, &addrlen, flags);
 	ss2zta(env, &ss, addr);
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 #endif
 
-int zts_setsockopt(int fd, int level, int optname, const void *optval, socklen_t optlen)
+int zts_setsockopt(int fd, int level, int optname, const void *optval,zts_socklen_t optlen)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_setsockopt(fd, level, optname, optval, optlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_setsockopt(fd, level, optname, optval, optlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
@@ -166,7 +183,7 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
 {
 	jclass c = env->GetObjectClass(optval);
 	if (!c) {
-		return ZTS_ERR_INVALID_OP;
+		return ZTS_ERR_SERVICE;
 	}
 	int optval_int = -1;
 
@@ -206,9 +223,12 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_setsockopt(
 }
 #endif
 
-int zts_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
+int zts_getsockopt(int fd, int level, int optname, void *optval, zts_socklen_t *optlen)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getsockopt(fd, level, optname, optval, optlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_getsockopt(fd, level, optname, optval, (socklen_t*)optlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
@@ -216,15 +236,15 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
 {
 	jclass c = env->GetObjectClass(optval);
 	if (!c) {
-		return ZTS_ERR_INVALID_OP;
+		return ZTS_ERR_SERVICE;
 	}
 	int optval_int = 0;
-	socklen_t optlen; // Intentionally not used
+	zts_socklen_t optlen; // Intentionally not used
 
 	int retval;
 
 	if (optname == SO_RCVTIMEO) {
-		struct timeval tv;
+		struct zts_timeval tv;
 		optlen = sizeof(tv);
 		retval = zts_getsockopt(fd, level, optname, &tv, &optlen);
 		// Convert seconds and microseconds back to milliseconds
@@ -261,91 +281,61 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getsockopt(
 }
 #endif
 
-int zts_getsockname(int fd, struct sockaddr *addr, socklen_t *addrlen)
+int zts_getsockname(int fd, struct zts_sockaddr *addr, zts_socklen_t *addrlen)
 {
 	if (!addr) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
+	if (*addrlen > (int)sizeof(struct zts_sockaddr_storage) || *addrlen < (int)sizeof(struct zts_sockaddr_in)) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getsockname(fd, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_getsockname(fd, (sockaddr*)addr, (socklen_t*)addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jboolean JNICALL Java_com_zerotier_libzt_ZeroTier_getsockname(JNIEnv *env, jobject thisObj,
 	jint fd, jobject addr)
 {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
-	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	zts_socklen_t addrlen = sizeof(struct zts_sockaddr_storage);
-	int retval =zts_getsockname(fd, (struct sockaddr *)&ss, &addrlen);
+	int retval = zts_getsockname(fd, (struct zts_sockaddr *)&ss, &addrlen);
 	ss2zta(env, &ss, addr);	 
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
-int zts_getpeername(int fd, struct sockaddr *addr, socklen_t *addrlen)
+int zts_getpeername(int fd, struct zts_sockaddr *addr, zts_socklen_t *addrlen)
 {
 	if (!addr) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	if (*addrlen > (int)sizeof(struct sockaddr_storage) || *addrlen < (int)sizeof(struct sockaddr_in)) {
+	if (*addrlen > (int)sizeof(struct zts_sockaddr_storage) || *addrlen < (int)sizeof(struct zts_sockaddr_in)) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_getpeername(fd, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_getpeername(fd, (sockaddr*)addr, (socklen_t*)addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_getpeername(JNIEnv *env, jobject thisObj,
 	jint fd, jobject addr)
 {
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
-	int retval = zts_getpeername(fd, (struct sockaddr *)&ss, (socklen_t *)sizeof(struct sockaddr_storage));
+	int retval = zts_getpeername(fd, (struct zts_sockaddr *)&ss, (zts_socklen_t*)sizeof(struct zts_sockaddr_storage));
 	ss2zta(env, &ss, addr);	 
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
-int zts_gethostname(char *name, size_t len)
-{
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
-}
-#ifdef SDK_JNI
-#endif
-
-int zts_sethostname(const char *name, size_t len)
-{
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_INVALID_OP : ZTS_ERR_INVALID_OP; // TODO
-}
-#ifdef SDK_JNI
-#endif
-
-/*
-struct hostent *zts_gethostbyname(const char *name)
-{
-	return (struct hostent *)((!_run_service || !_run_lwip_tcpip) ? NULL : NULL);
-	// TODO: Test thread safety
-	char buf[256];
-	int buflen = 256;
-	int h_err = 0;
-	struct hostent hret;
-	struct hostent **result = NULL;
-	int err = 0;
-	if ((err = lwip_gethostbyname_r(name, &hret, buf, buflen, result, &h_err)) != 0) {
-		DEBUG_ERROR("err = %d", err);
-		DEBUG_ERROR("h_err = %d", h_err);
-		errno = h_err;
-		return NULL; // failure
-	}
-	return *result;
-	
-	return lwip_gethostbyname(name);
-}
-#ifdef SDK_JNI
-#endif
-*/
-
 int zts_close(int fd)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_close(fd);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_close(fd);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_close(
@@ -355,22 +345,25 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_close(
 }
 #endif
 
-int zts_select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds,
+int zts_select(int nfds, zts_fd_set *readfds, zts_fd_set *writefds, zts_fd_set *exceptfds,
-	struct timeval *timeout)
+	struct zts_timeval *timeout)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_select(nfds, readfds, writefds, exceptfds, timeout);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_select(nfds, (fd_set*)readfds, (fd_set*)writefds, (fd_set*)exceptfds, (timeval*)timeout);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_select(JNIEnv *env, jobject thisObj,
 	jint nfds, jobject readfds, jobject writefds, jobject exceptfds, jint timeout_sec, jint timeout_usec)
 {
-	struct timeval _timeout;
+	struct zts_timeval _timeout;
 	_timeout.tv_sec  = timeout_sec;
 	_timeout.tv_usec = timeout_usec;
-	fd_set _readfds, _writefds, _exceptfds;   		
+	zts_fd_set _readfds, _writefds, _exceptfds;
-	fd_set *r = NULL;
+	zts_fd_set *r = NULL;
-	fd_set *w = NULL;
+	zts_fd_set *w = NULL;
-	fd_set *e = NULL;
+	zts_fd_set *e = NULL;
 	if (readfds) {
 		r = &_readfds;
 		ztfdset2fdset(env, nfds, readfds, &_readfds);
@@ -411,7 +404,10 @@ int zts_fcntl(int fd, int cmd, int flags)
 		translated_flags = 1;
 	}
 #endif
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_fcntl(fd, cmd, translated_flags);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_fcntl(fd, cmd, translated_flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_fcntl(
@@ -422,12 +418,24 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_fcntl(
 }
 #endif
 
+// TODO: JNI version
+int zts_poll(struct zts_pollfd *fds, nfds_t nfds, int timeout)
+{
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_poll((pollfd*)fds, nfds, timeout);
+}
+
 int zts_ioctl(int fd, unsigned long request, void *argp)
 {
 	if (!argp) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_ioctl(fd, request, argp);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_ioctl(fd, request, argp);
 }
 #ifdef SDK_JNI
 JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
@@ -435,13 +443,12 @@ JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
 {
 	int retval = ZTS_ERR_OK;
 	if (request == FIONREAD) {
-		// DEBUG_ERROR("FIONREAD");
 		int bytesRemaining = 0;
 		retval = zts_ioctl(fd, request, &bytesRemaining);	
 		// set value in general object
 		jclass c = env->GetObjectClass(argp);
 		if (!c) {
-			return ZTS_ERR_INVALID_ARG;
+			return ZTS_ERR_ARG;
 		}
 		jfieldID fid = env->GetFieldID(c, "integer", "I");
 		env->SetIntField(argp, fid, bytesRemaining);
@@ -449,7 +456,6 @@ JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
 	if (request == FIONBIO) {
 		// TODO: double check
 		int meaninglessVariable = 0;
-		// DEBUG_ERROR("FIONBIO");
 		retval = zts_ioctl(fd, request, &meaninglessVariable);		
 	}
 	return retval > -1 ? retval : -(zts_errno);
@@ -459,9 +465,12 @@ JNIEXPORT int JNICALL Java_com_zerotier_libzt_ZeroTier_ioctl(
 ssize_t zts_send(int fd, const void *buf, size_t len, int flags)
 {
 	if (!buf || len <= 0) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_send(fd, buf, len, flags);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_send(fd, buf, len, flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_send(
@@ -475,25 +484,28 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_send(
 #endif
 
 ssize_t zts_sendto(int fd, const void *buf, size_t len, int flags, 
-	const struct sockaddr *addr, socklen_t addrlen)
+	const struct zts_sockaddr *addr,zts_socklen_t addrlen)
 {
 	if (!addr || !buf || len <= 0) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	if (addrlen > (int)sizeof(struct sockaddr_storage) || addrlen < (int)sizeof(struct sockaddr_in)) {
+	if (addrlen > (int)sizeof(struct zts_sockaddr_storage) || addrlen < (int)sizeof(struct zts_sockaddr_in)) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_sendto(fd, buf, len, flags, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_sendto(fd, buf, len, flags, (sockaddr*)addr, addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(
 	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, jint flags, jobject addr)
 {
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
 	zta2ss(env, &ss, addr);
-	socklen_t addrlen = ss.ss_family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
+	zts_socklen_t addrlen = ss.ss_family == ZTS_AF_INET ? sizeof(struct zts_sockaddr_in) : sizeof(struct zts_sockaddr_in6);
-	int retval = zts_sendto(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, addrlen);
+	int retval = zts_sendto(fd, data, env->GetArrayLength(buf), flags, (struct zts_sockaddr *)&ss, addrlen);
 	env->ReleasePrimitiveArrayCritical(buf, data, 0);	 
 	return retval > -1 ? retval : -(zts_errno);
 }
@@ -501,7 +513,10 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_sendto(
 
 ssize_t zts_sendmsg(int fd, const struct msghdr *msg, int flags)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_sendmsg(fd, msg, flags);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_sendmsg(fd, msg, flags);
 }
 #ifdef SDK_JNI
 #endif
@@ -509,9 +524,12 @@ ssize_t zts_sendmsg(int fd, const struct msghdr *msg, int flags)
 ssize_t zts_recv(int fd, void *buf, size_t len, int flags)
 {
 	if (!buf) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_recv(fd, buf, len, flags);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_recv(fd, buf, len, flags);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recv(JNIEnv *env, jobject thisObj,
@@ -525,49 +543,64 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recv(JNIEnv *env, jobjec
 #endif
 
 ssize_t zts_recvfrom(int fd, void *buf, size_t len, int flags, 
-	struct sockaddr *addr, socklen_t *addrlen)
+	struct zts_sockaddr *addr, zts_socklen_t *addrlen)
 {
 	if (!buf) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_recvfrom(fd, buf, len, flags, addr, addrlen);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_recvfrom(fd, buf, len, flags, (sockaddr*)addr, (socklen_t*)addrlen);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_recvfrom(
 	JNIEnv *env, jobject thisObj, jint fd, jbyteArray buf, jint flags, jobject addr)
 {
-	socklen_t addrlen = sizeof(struct sockaddr_storage);
+	zts_socklen_t addrlen = sizeof(struct zts_sockaddr_storage);
-	struct sockaddr_storage ss;
+	struct zts_sockaddr_storage ss;
 	void *data = env->GetPrimitiveArrayCritical(buf, NULL);
-	int retval = zts_recvfrom(fd, data, env->GetArrayLength(buf), flags, (struct sockaddr *)&ss, &addrlen);
+	int retval = zts_recvfrom(fd, data, env->GetArrayLength(buf), flags, (struct zts_sockaddr *)&ss, &addrlen);
 	env->ReleasePrimitiveArrayCritical(buf, data, 0);
 	ss2zta(env, &ss, addr);	 
 	return retval > -1 ? retval : -(zts_errno);
 }
 #endif
 
+// TODO: JNI version
 ssize_t zts_recvmsg(int fd, struct msghdr *msg, int flags)
 {
-	// Not currently implemented by stack
+	if (!msg) {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : ZTS_ERR_GENERAL;
+		return ZTS_ERR_ARG;
+	}
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_recvmsg(fd, msg, flags);
 }
 #ifdef SDK_JNI
 #endif
 
-int zts_read(int fd, void *buf, size_t len)
+ssize_t zts_read(int fd, void *buf, size_t len)
 {
 	if (!buf) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_read(fd, buf, len);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_read(fd, buf, len);
 }
-int zts_read_offset(int fd, void *buf, size_t offset, size_t len)
+ssize_t zts_read_offset(int fd, void *buf, size_t offset, size_t len)
 {
 	if (!buf) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
+	}
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
 	}
 	char *cbuf = (char*)buf;
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_read(fd, &(cbuf[offset]), len);
+	return lwip_read(fd, &(cbuf[offset]), len);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read(JNIEnv *env, jobject thisObj,
@@ -596,12 +629,24 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_read_1length(JNIEnv *env
 }
 #endif
 
-int zts_write(int fd, const void *buf, size_t len)
+// TODO: JNI version
+ssize_t zts_readv(int s, const struct zts_iovec *iov, int iovcnt)
+{
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_readv(s, (iovec*)iov, iovcnt);
+}
+
+ssize_t zts_write(int fd, const void *buf, size_t len)
 {
 	if (!buf || len <= 0) {
-		return ZTS_ERR_INVALID_ARG;
+		return ZTS_ERR_ARG;
 	}
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_write(fd, buf, len);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_write(fd, buf, len);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write__IB(JNIEnv *env, jobject thisObj,
@@ -628,9 +673,21 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_write_1byte(JNIEnv *env,
 }
 #endif
 
+// TODO: JNI version
+ssize_t zts_writev(int fd, const struct zts_iovec *iov, int iovcnt)
+{
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_writev(fd, (iovec*)iov, iovcnt);
+}
+
 int zts_shutdown(int fd, int how)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : lwip_shutdown(fd, how);
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return lwip_shutdown(fd, how);
 }
 #ifdef SDK_JNI
 JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_shutdown(
@@ -640,42 +697,213 @@ JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_shutdown(
 }
 #endif
 
-int zts_add_dns_nameserver(struct sockaddr *addr)
+int zts_add_dns_nameserver(struct zts_sockaddr *addr)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : -1; // TODO
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return ZTS_ERR_SERVICE; // TODO
 }
 #ifdef SDK_JNI
 #endif
 
-int zts_del_dns_nameserver(struct sockaddr *addr)
+int zts_del_dns_nameserver(struct zts_sockaddr *addr)
 {
-	return (!_run_service || !_run_lwip_tcpip) ? ZTS_ERR_SERVICE : -1; // TODO
+	if (!(_serviceStateFlags & ZTS_STATE_NET_SERVICE_RUNNING)) {
+		return ZTS_ERR_SERVICE;
+	}
+	return ZTS_ERR_SERVICE; // TODO
 }
 #ifdef SDK_JNI
 #endif
 
+uint16_t zts_htons(uint16_t n)
+{
+	return lwip_htons(n);
+}
+
+uint32_t zts_htonl(uint32_t n)
+{
+	return lwip_htonl(n);
+}
+
+uint16_t zts_ntohs(uint16_t n)
+{
+	return lwip_htons(n);
+}
+
+uint32_t zts_ntohl(uint32_t n)
+{
+	return lwip_htonl(n);
+}
+
+const char *zts_inet_ntop(int af, const void *src, char *dst,zts_socklen_t size)
+{
+	return lwip_inet_ntop(af,src,dst,size);
+}
+
+int zts_inet_pton(int af, const char *src, void *dst)
+{
+	return lwip_inet_pton(af,src,dst);
+}
+
+uint32_t zts_inet_addr(const char *cp)
+{
+	return ipaddr_addr(cp);
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// Statistics                                                               //
+//////////////////////////////////////////////////////////////////////////////
+
+extern struct stats_ lwip_stats;
+
+int zts_get_all_stats(struct zts_stats *statsDest)
+{
+#if LWIP_STATS
+	if (!statsDest) {
+		return ZTS_ERR_ARG;
+	}
+	memset(statsDest, 0, sizeof(struct zts_stats));
+	// Copy lwIP stats
+	memcpy(&(statsDest->link), &(lwip_stats.link), sizeof(struct stats_proto));
+	memcpy(&(statsDest->etharp), &(lwip_stats.etharp), sizeof(struct stats_proto));
+	memcpy(&(statsDest->ip_frag), &(lwip_stats.ip_frag), sizeof(struct stats_proto));
+	memcpy(&(statsDest->ip), &(lwip_stats.ip), sizeof(struct stats_proto));
+	memcpy(&(statsDest->icmp), &(lwip_stats.icmp), sizeof(struct stats_proto));
+	//memcpy(&(statsDest->igmp), &(lwip_stats.igmp), sizeof(struct stats_igmp));
+	memcpy(&(statsDest->udp), &(lwip_stats.udp), sizeof(struct stats_proto));
+	memcpy(&(statsDest->tcp), &(lwip_stats.tcp), sizeof(struct stats_proto));
+	// mem omitted
+	// memp omitted
+	memcpy(&(statsDest->sys), &(lwip_stats.sys), sizeof(struct stats_sys));
+	memcpy(&(statsDest->ip6), &(lwip_stats.ip6), sizeof(struct stats_proto));
+	memcpy(&(statsDest->icmp6), &(lwip_stats.icmp6), sizeof(struct stats_proto));
+	memcpy(&(statsDest->ip6_frag), &(lwip_stats.ip6_frag), sizeof(struct stats_proto));
+	memcpy(&(statsDest->mld6), &(lwip_stats.mld6), sizeof(struct stats_igmp));
+	memcpy(&(statsDest->nd6), &(lwip_stats.nd6), sizeof(struct stats_proto));
+	memcpy(&(statsDest->ip_frag), &(lwip_stats.ip_frag), sizeof(struct stats_proto));
+	// mib2 omitted
+	// Copy ZT stats
+	// ...
+	return ZTS_ERR_OK;
+#else
+	return ZTS_ERR_NO_RESULT;
+#endif
+}
 #ifdef SDK_JNI
-void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, fd_set *dest_fd_set)
+	// No implementation for JNI
+#endif
+
+int zts_get_protocol_stats(int protocolType, void *protoStatsDest)
+{
+#if LWIP_STATS
+	if (!protoStatsDest) {
+		return ZTS_ERR_ARG;
+	}
+	memset(protoStatsDest, 0, sizeof(struct stats_proto));
+	switch (protocolType)
+	{
+		case ZTS_STATS_PROTOCOL_LINK:
+			memcpy(protoStatsDest, &(lwip_stats.link), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_ETHARP:
+			memcpy(protoStatsDest, &(lwip_stats.etharp), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_IP:
+			memcpy(protoStatsDest, &(lwip_stats.ip), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_UDP:
+			memcpy(protoStatsDest, &(lwip_stats.udp), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_TCP:
+			memcpy(protoStatsDest, &(lwip_stats.tcp), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_ICMP:
+			memcpy(protoStatsDest, &(lwip_stats.icmp), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_IP_FRAG:
+			memcpy(protoStatsDest, &(lwip_stats.ip_frag), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_IP6:
+			memcpy(protoStatsDest, &(lwip_stats.ip6), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_ICMP6:
+			memcpy(protoStatsDest, &(lwip_stats.icmp6), sizeof(struct stats_proto));
+			break;
+		case ZTS_STATS_PROTOCOL_IP6_FRAG:
+			memcpy(protoStatsDest, &(lwip_stats.ip6_frag), sizeof(struct stats_proto));
+			break;
+		default:
+			return ZTS_ERR_ARG;
+	}
+	return ZTS_ERR_OK;
+#else
+	return ZTS_ERR_NO_RESULT;
+#endif
+}
+#ifdef SDK_JNI
+JNIEXPORT jint JNICALL Java_com_zerotier_libzt_ZeroTier_get_1protocol_1stats(
+	JNIEnv *env, jobject thisObj, jint protocolType, jobject protoStatsObj)
+{
+	struct stats_proto stats;
+	int retval = zts_get_protocol_stats(protocolType, &stats);
+	// Copy stats into Java object
+	jclass c = env->GetObjectClass(protoStatsObj);
+	if (!c) {
+		return ZTS_ERR_ARG;
+	}
+	jfieldID fid;
+	fid = env->GetFieldID(c, "xmit", "I");
+	env->SetIntField(protoStatsObj, fid, stats.xmit);
+	fid = env->GetFieldID(c, "recv", "I");
+	env->SetIntField(protoStatsObj, fid, stats.recv);
+	fid = env->GetFieldID(c, "fw", "I");
+	env->SetIntField(protoStatsObj, fid, stats.fw);
+	fid = env->GetFieldID(c, "drop", "I");
+	env->SetIntField(protoStatsObj, fid, stats.drop);
+	fid = env->GetFieldID(c, "chkerr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.chkerr);
+	fid = env->GetFieldID(c, "lenerr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.lenerr);
+	fid = env->GetFieldID(c, "memerr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.memerr);
+	fid = env->GetFieldID(c, "rterr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.rterr);
+	fid = env->GetFieldID(c, "proterr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.proterr);
+	fid = env->GetFieldID(c, "opterr", "I");
+	env->SetIntField(protoStatsObj, fid, stats.opterr);
+	fid = env->GetFieldID(c, "err", "I");
+	env->SetIntField(protoStatsObj, fid, stats.err);
+	fid = env->GetFieldID(c, "cachehit", "I");
+	env->SetIntField(protoStatsObj, fid, stats.cachehit);
+	return retval;
+}
+#endif
+
+#ifdef SDK_JNI
+void ztfdset2fdset(JNIEnv *env, int nfds, jobject src_ztfd_set, zts_fd_set *dest_fd_set)
 {
 	jclass c = env->GetObjectClass(src_ztfd_set);
 	if (!c) {
 		return;
 	}
-	FD_ZERO(dest_fd_set);
+	ZTS_FD_ZERO(dest_fd_set);
 	jfieldID fid = env->GetFieldID(c, "fds_bits", "[B");
 	jobject fdData = env->GetObjectField (src_ztfd_set, fid);
 	jbyteArray * arr = reinterpret_cast<jbyteArray*>(&fdData);
 	char *data = (char*)env->GetByteArrayElements(*arr, NULL);
 	for (int i=0; i<nfds; i++) {
 		if (data[i] == 0x01)  {
-			FD_SET(i, dest_fd_set);
+			ZTS_FD_SET(i, dest_fd_set);
 		}
 	}
 	env->ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
 	return;
 }
 
-void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_set)
+void fdset2ztfdset(JNIEnv *env, int nfds, zts_fd_set *src_fd_set, jobject dest_ztfd_set)
 {
 	jclass c = env->GetObjectClass(dest_ztfd_set);
 	if (!c) {
@@ -686,7 +914,7 @@ void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_
 	jbyteArray * arr = reinterpret_cast<jbyteArray*>(&fdData);
 	char *data = (char*)env->GetByteArrayElements(*arr, NULL);
 	for (int i=0; i<nfds; i++) {
-		if (FD_ISSET(i, src_fd_set)) {
+		if (ZTS_FD_ISSET(i, src_fd_set)) {
 			data[i] = 0x01;
 		}
 	}
@@ -698,15 +926,15 @@ void fdset2ztfdset(JNIEnv *env, int nfds, fd_set *src_fd_set, jobject dest_ztfd_
 // Helpers (for moving data across the JNI barrier)                         //
 //////////////////////////////////////////////////////////////////////////////
 
-void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
+void ss2zta(JNIEnv *env, struct zts_sockaddr_storage *ss, jobject addr)
 {
 	jclass c = env->GetObjectClass(addr);
 	if (!c) {
 		return;
 	}
-	if(ss->ss_family == AF_INET)
+	if(ss->ss_family == ZTS_AF_INET)
 	{
-		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
+		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)ss;
 		jfieldID fid = env->GetFieldID(c, "_port", "I");
 		env->SetIntField(addr, fid, lwip_ntohs(in4->sin_port));
 		fid = env->GetFieldID(c,"_family", "I");
@@ -720,9 +948,9 @@ void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 
 		return;
 	}
-	if(ss->ss_family == AF_INET6)
+	if(ss->ss_family == ZTS_AF_INET6)
 	{
-		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)ss;
 		jfieldID fid = env->GetFieldID(c, "_port", "I");
 		env->SetIntField(addr, fid, lwip_ntohs(in6->sin6_port));
 		fid = env->GetFieldID(c,"_family", "I");
@@ -737,7 +965,7 @@ void ss2zta(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	}
 }
 
-void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
+void zta2ss(JNIEnv *env, struct zts_sockaddr_storage *ss, jobject addr)
 {
 	jclass c = env->GetObjectClass(addr);
 	if (!c) {
@@ -745,12 +973,12 @@ void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 	}
 	jfieldID fid = env->GetFieldID(c, "_family", "I");
 	int family = env->GetIntField(addr, fid);
-	if (family == AF_INET)
+	if (family == ZTS_AF_INET)
 	{
-		struct sockaddr_in *in4 = (struct sockaddr_in*)ss;
+		struct zts_sockaddr_in *in4 = (struct zts_sockaddr_in*)ss;
 		fid = env->GetFieldID(c, "_port", "I");
 		in4->sin_port = lwip_htons(env->GetIntField(addr, fid));
-		in4->sin_family = AF_INET;
+		in4->sin_family = ZTS_AF_INET;
 		fid = env->GetFieldID(c, "_ip4", "[B");
 		jobject ipData = env->GetObjectField (addr, fid);
 		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);
@@ -759,13 +987,13 @@ void zta2ss(JNIEnv *env, struct sockaddr_storage *ss, jobject addr)
 		env->ReleaseByteArrayElements(*arr, (jbyte*)data, 0);
 		return;
 	}
-	if (family == AF_INET6)
+	if (family == ZTS_AF_INET6)
 	{
-		struct sockaddr_in6 *in6 = (struct sockaddr_in6*)ss;
+		struct zts_sockaddr_in6 *in6 = (struct zts_sockaddr_in6*)ss;
 		jfieldID fid = env->GetFieldID(c, "_port", "I");
 		in6->sin6_port = lwip_htons(env->GetIntField(addr, fid));
 		fid = env->GetFieldID(c,"_family", "I");
-		in6->sin6_family = AF_INET6;
+		in6->sin6_family = ZTS_AF_INET6;
 		fid = env->GetFieldID(c, "_ip6", "[B");
 		jobject ipData = env->GetObjectField (addr, fid);
 		jbyteArray * arr = reinterpret_cast<jbyteArray*>(&ipData);

src/VirtualTap.cpp

@@ -14,28 +14,41 @@
 /**
  * @file
  *
- * Virtual Ethernet tap device
+ * Virtual ethernet tap device and combined network stack driver
  */
 
-#include "VirtualTap.hpp"
+#include "MAC.hpp"
-#include "Phy.hpp"
-#include "Node.hpp"
-//#include "OSUtils.hpp"
-
-#include "Service.hpp"
 #include "Mutex.hpp"
-#include "lwipDriver.hpp"
+#include "InetAddress.hpp"
-#include "ZeroTier.h"
+#include "MulticastGroup.hpp"
 
-#ifdef _MSC_VER
+#include "lwip/netif.h"
+#include "lwip/etharp.h"
+#include "lwip/sys.h"
+#include "lwip/ethip6.h"
+#include "lwip/tcpip.h"
+#include "netif/ethernet.h"
+
+#ifdef LWIP_STATS
+#include "lwip/stats.h"
+#endif
+
+#include "VirtualTap.hpp"
+#include "ZeroTierSockets.h"
+#include "Events.hpp"
+#include "Debug.hpp"
+
+#if defined(__WINDOWS__)
+#include <time.h>
 #include "Synchapi.h"
 #endif
 
+#define ZTS_TAP_THREAD_POLLING_INTERVAL 50
+#define LWIP_DRIVER_LOOP_INTERVAL       250
+
 namespace ZeroTier {
 
-class VirtualTap;
+extern void _enqueueEvent(int16_t eventCode, void *arg = NULL);
-extern OneService *service;
-extern void postEvent(int eventCode, void *arg);
 
 /**
  * A virtual tap device. The ZeroTier core service creates one of these for each
@@ -66,7 +79,7 @@ VirtualTap::VirtualTap(
 	memset(vtap_full_name, 0, sizeof(vtap_full_name));
 	snprintf(vtap_full_name, sizeof(vtap_full_name), "libzt%llx", (unsigned long long)_nwid);
 	_dev = vtap_full_name;
-#ifndef _WIN32
+#ifndef __WINDOWS__
 	::pipe(_shutdownSignalPipe);
 #endif
 	// Start virtual tap thread and stack I/O loops
@@ -77,18 +90,18 @@ VirtualTap::~VirtualTap()
 {
 	struct zts_network_details *nd = new zts_network_details;
 	nd->nwid = _nwid;
-	postEvent(ZTS_EVENT_NETWORK_DOWN, (void*)nd);
+	_enqueueEvent(ZTS_EVENT_NETWORK_DOWN, (void*)nd);
 	_run = false;
-#ifndef _WIN32
+#ifndef __WINDOWS__
 	::write(_shutdownSignalPipe[1],"\0",1);
 #endif
 	_phy.whack();
-	lwip_remove_netif(netif4);
+	_lwip_remove_netif(netif4);
 	netif4 = NULL;
-	lwip_remove_netif(netif6);
+	_lwip_remove_netif(netif6);
 	netif6 = NULL;
 	Thread::join(_thread);
-#ifndef _WIN32
+#ifndef __WINDOWS__
 	::close(_shutdownSignalPipe[0]);
 	::close(_shutdownSignalPipe[1]);
 #endif
@@ -156,7 +169,7 @@ bool VirtualTap::addIp(const InetAddress &ip)
 		return false;
 	}
 	if (std::find(_ips.begin(),_ips.end(),ip) == _ips.end()) {
-		lwip_init_interface((void*)this, ip);
+		_lwip_init_interface((void*)this, ip);
 		// TODO: Add ZTS_EVENT_ADDR_NEW ?
 		_ips.push_back(ip);
 		// Send callback message
@@ -165,12 +178,12 @@ bool VirtualTap::addIp(const InetAddress &ip)
 		if (ip.isV4()) {
 			struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr);
 			memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4);
-			postEvent(ZTS_EVENT_ADDR_ADDED_IP4, (void*)ad);
+			_enqueueEvent(ZTS_EVENT_ADDR_ADDED_IP4, (void*)ad);
 		}
 		if (ip.isV6()) {
 			struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr);
 			memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16);
-			postEvent(ZTS_EVENT_ADDR_ADDED_IP6, (void*)ad);
+			_enqueueEvent(ZTS_EVENT_ADDR_ADDED_IP6, (void*)ad);
 		}
 		std::sort(_ips.begin(),_ips.end());
 	}
@@ -187,14 +200,14 @@ bool VirtualTap::removeIp(const InetAddress &ip)
 		if (ip.isV4()) {
 			struct sockaddr_in *in4 = (struct sockaddr_in*)&(ad->addr);
 			memcpy(&(in4->sin_addr.s_addr), ip.rawIpData(), 4);
-			postEvent(ZTS_EVENT_ADDR_REMOVED_IP4, (void*)ad);
+			_enqueueEvent(ZTS_EVENT_ADDR_REMOVED_IP4, (void*)ad);
 			// FIXME: De-register from network stack
 		}
 		if (ip.isV6()) {
 			// FIXME: De-register from network stack
 			struct sockaddr_in6 *in6 = (struct sockaddr_in6*)&(ad->addr);
 			memcpy(&(in6->sin6_addr.s6_addr), ip.rawIpData(), 16);
-			postEvent(ZTS_EVENT_ADDR_REMOVED_IP6, (void*)ad);
+			_enqueueEvent(ZTS_EVENT_ADDR_REMOVED_IP6, (void*)ad);
 		}
 		_ips.erase(i);
 	}
@@ -211,7 +224,7 @@ void VirtualTap::put(const MAC &from,const MAC &to,unsigned int etherType,
 	const void *data,unsigned int len)
 {
 	if (len <= _mtu && _enabled) {
-		lwip_eth_rx(this, from, to, etherType, data, len);
+		_lwip_eth_rx(this, from, to, etherType, data, len);
 	}
 }
 
@@ -220,19 +233,6 @@ std::string VirtualTap::deviceName() const
 	return _dev;
 }
 
-std::string VirtualTap::nodeId() const
-{
-	if (service) {
-		char id[16];
-		memset(id, 0, sizeof(id));
-		sprintf(id, "%llx", (unsigned long long)((OneService *)service)->getNode()->address());
-		return std::string(id);
-	}
-	else {
-		return std::string("----------");
-	}
-}
-
 void VirtualTap::setFriendlyName(const char *friendlyName)
 {
 	DEBUG_INFO("%s", friendlyName);
@@ -290,7 +290,7 @@ void VirtualTap::threadMain()
 		if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) {
 			break;
 		}
-#if defined(_WIN32)
+#if defined(__WINDOWS__)
 		Sleep(ZTS_TAP_THREAD_POLLING_INTERVAL);
 #else
 		struct timespec sleepValue = {0};
@@ -300,11 +300,6 @@ void VirtualTap::threadMain()
 	}
 }
 
-void VirtualTap::Housekeeping()
-{
-    //
-}
-
 void VirtualTap::phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *local_address,
 	const struct sockaddr *from,void *data,unsigned long len) {}
 void VirtualTap::phyOnTcpConnect(PhySocket *sock,void **uptr,bool success) {}
@@ -313,5 +308,461 @@ void VirtualTap::phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,v
 void VirtualTap::phyOnTcpClose(PhySocket *sock,void **uptr) {}
 void VirtualTap::phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 void VirtualTap::phyOnTcpWritable(PhySocket *sock,void **uptr) {}
+void VirtualTap::phyOnUnixClose(PhySocket *sock,void **uptr) {}
 
-} // namespace ZeroTier
+//////////////////////////////////////////////////////////////////////////////
+// Netif driver code for lwIP network stack                                 //
+//////////////////////////////////////////////////////////////////////////////
+
+bool _has_exited = false;
+
+// Used to generate enumerated lwIP interface names
+int netifCount = 0;
+
+// Lock to guard access to network stack state changes
+Mutex stackLock;
+
+// Callback for when the TCPIP thread has been successfully started
+static void _tcpip_init_done(void *arg)
+{
+	sys_sem_t *sem;
+	sem = (sys_sem_t *)arg;
+	_setState(ZTS_STATE_STACK_RUNNING);
+	_enqueueEvent(ZTS_EVENT_STACK_UP);
+	sys_sem_signal(sem);
+}
+
+static void _main_lwip_driver_loop(void *arg)
+{
+#if defined(__linux__)
+	pthread_setname_np(pthread_self(), ZTS_LWIP_DRIVER_THREAD_NAME);
+#endif
+#if defined(__APPLE__)
+	pthread_setname_np(ZTS_LWIP_DRIVER_THREAD_NAME);
+#endif
+	sys_sem_t sem;
+	LWIP_UNUSED_ARG(arg);
+	if (sys_sem_new(&sem, 0) != ERR_OK) {
+		DEBUG_ERROR("failed to create semaphore");
+	}
+	tcpip_init(_tcpip_init_done, &sem);
+	sys_sem_wait(&sem);
+	// Main loop
+	while(_getState(ZTS_STATE_STACK_RUNNING)) {
+		zts_delay_ms(LWIP_DRIVER_LOOP_INTERVAL);
+	}
+	_has_exited = true;
+	_enqueueEvent(ZTS_EVENT_STACK_DOWN);
+}
+
+bool _lwip_is_up()
+{
+	Mutex::Lock _l(stackLock);
+	return _getState(ZTS_STATE_STACK_RUNNING);
+}
+
+bool _lwip_has_previously_shutdown()
+{
+	Mutex::Lock _l(stackLock);
+	return _has_exited;
+}
+
+void _lwip_driver_init()
+{
+	if (_lwip_is_up()) {
+		return;
+	}
+	if (_lwip_has_previously_shutdown()) {
+		return;
+	}
+	Mutex::Lock _l(stackLock);
+#if defined(__WINDOWS__)
+	sys_init(); // Required for win32 init of critical sections
+#endif
+	sys_thread_new(ZTS_LWIP_DRIVER_THREAD_NAME, _main_lwip_driver_loop,
+		NULL, DEFAULT_THREAD_STACKSIZE, DEFAULT_THREAD_PRIO);
+}
+
+void _lwip_driver_shutdown()
+{
+	if (_lwip_has_previously_shutdown()) {
+		return;
+	}
+	Mutex::Lock _l(stackLock);
+	// Set flag to stop sending frames into the core
+	_clrState(ZTS_STATE_STACK_RUNNING);
+	// Wait until the main lwIP thread has exited
+	while (!_has_exited) { zts_delay_ms(LWIP_DRIVER_LOOP_INTERVAL); }
+	/*
+	if (tcpip_shutdown() == ERR_OK) {
+		sys_timeouts_free();
+	}
+	*/
+}
+
+void _lwip_remove_netif(void *netif)
+{
+	if (!netif) {
+		return;
+	}
+	struct netif *n = (struct netif*)netif;
+	LOCK_TCPIP_CORE();
+	netif_remove(n);
+	netif_set_down(n);
+	netif_set_link_down(n);
+	UNLOCK_TCPIP_CORE();
+}
+
+err_t _lwip_eth_tx(struct netif *n, struct pbuf *p)
+{
+	if (!n) {
+		return ERR_IF;
+	}
+	struct pbuf *q;
+	char buf[ZT_MAX_MTU+32];
+	char *bufptr;
+	int totalLength = 0;
+
+	VirtualTap *tap = (VirtualTap*)n->state;
+	bufptr = buf;
+	for (q = p; q != NULL; q = q->next) {
+		memcpy(bufptr, q->payload, q->len);
+		bufptr += q->len;
+		totalLength += q->len;
+	}
+	struct eth_hdr *ethhdr;
+	ethhdr = (struct eth_hdr *)buf;
+
+	MAC src_mac;
+	MAC dest_mac;
+	src_mac.setTo(ethhdr->src.addr, 6);
+	dest_mac.setTo(ethhdr->dest.addr, 6);
+
+	char *data = buf + sizeof(struct eth_hdr);
+	int len = totalLength - sizeof(struct eth_hdr);
+	int proto = Utils::ntoh((uint16_t)ethhdr->type);
+	tap->_handler(tap->_arg, NULL, tap->_nwid, src_mac, dest_mac, proto, 0, data, len);
+	if (ZT_MSG_TRANSFER == true) {
+		char flagbuf[32];
+		memset(&flagbuf, 0, 32);
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
+		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			ethhdr->dest.addr[0], ethhdr->dest.addr[1], ethhdr->dest.addr[2],
+			ethhdr->dest.addr[3], ethhdr->dest.addr[4], ethhdr->dest.addr[5]);
+		MAC mac;
+		mac.setTo(ethhdr->dest.addr, 6);
+		mac.toAddress(tap->_nwid).toString(nodeBuf);
+		/*
+		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] ethertype=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
+			Utils::ntoh(ethhdr->type), flagbuf);
+		*/
+	}
+	return ERR_OK;
+}
+
+void _lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
+	const void *data, unsigned int len)
+{
+#ifdef LWIP_STATS
+	stats_display();
+#endif
+	if (!_getState(ZTS_STATE_STACK_RUNNING)) {
+		return;
+	}
+	struct pbuf *p,*q;
+	struct eth_hdr ethhdr;
+	from.copyTo(ethhdr.src.addr, 6);
+	to.copyTo(ethhdr.dest.addr, 6);
+	ethhdr.type = Utils::hton((uint16_t)etherType);
+
+	if (ZT_MSG_TRANSFER == true) {
+		char flagbuf[32];
+		memset(&flagbuf, 0, 32);
+		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
+		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			ethhdr.dest.addr[0], ethhdr.dest.addr[1], ethhdr.dest.addr[2],
+			ethhdr.dest.addr[3], ethhdr.dest.addr[4], ethhdr.dest.addr[5]);
+		MAC mac;
+		mac.setTo(ethhdr.src.addr, 6);
+		mac.toAddress(tap->_nwid).toString(nodeBuf);
+		/*
+		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] ethertype=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
+			Utils::ntoh(ethhdr.type), flagbuf);
+		*/
+	}
+
+	p = pbuf_alloc(PBUF_RAW, (uint16_t)len+sizeof(struct eth_hdr), PBUF_RAM);
+	if (!p) {
+		DEBUG_ERROR("dropped packet: unable to allocate memory for pbuf");
+		return;
+	}
+	// First pbuf gets ethernet header at start
+	q = p;
+	if (q->len < sizeof(ethhdr)) {
+		pbuf_free(p);
+		p = NULL;
+		DEBUG_ERROR("dropped packet: first pbuf smaller than ethernet header");
+		return;
+	}
+	// Copy frame data into pbuf
+	const char *dataptr = reinterpret_cast<const char *>(data);
+	memcpy(q->payload,&ethhdr,sizeof(ethhdr));
+	int remainingPayloadSpace = q->len - sizeof(ethhdr);
+	memcpy((char*)q->payload + sizeof(ethhdr),dataptr,remainingPayloadSpace);
+	dataptr += remainingPayloadSpace;
+	// Remaining pbufs (if any) get rest of data
+	while ((q = q->next)) {
+		memcpy(q->payload,dataptr,q->len);
+		dataptr += q->len;
+	}
+	// Feed packet into stack
+	int err;
+
+	if (Utils::ntoh(ethhdr.type) == 0x800 || Utils::ntoh(ethhdr.type) == 0x806) {
+		if ((err = ((struct netif *)tap->netif4)->input(p, (struct netif *)tap->netif4)) != ERR_OK) {
+			DEBUG_ERROR("packet input error (%d)", err);
+			pbuf_free(p);
+		}
+	}
+	if (Utils::ntoh(ethhdr.type) == 0x86DD) {
+		if ((err = ((struct netif *)tap->netif6)->input(p, (struct netif *)tap->netif6)) != ERR_OK) {
+			DEBUG_ERROR("packet input error (%d)", err);
+			pbuf_free(p);
+		}
+	}
+}
+
+/*
+static void print_netif_info(struct netif *n) {
+	DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n",
+		n,
+		n->name[0],
+		n->name[1],
+		n->mtu,
+		n->output,
+		n->output_ip6,
+		n->hwaddr[0],
+		n->hwaddr[1],
+		n->hwaddr[2],
+		n->hwaddr[3],
+		n->hwaddr[4],
+		n->hwaddr[5],
+		n->hwaddr_len,
+		n->state,
+		n->flags
+	);
+}
+*/
+
+bool _lwip_is_netif_up(void *n)
+{
+	if (!n) {
+		return false;
+	}
+	LOCK_TCPIP_CORE();
+	bool result = netif_is_up((struct netif*)n);
+	UNLOCK_TCPIP_CORE();
+	return result;
+}
+
+/**
+ * Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
+ */
+#if LWIP_NETIF_REMOVE_CALLBACK
+static void _netif_remove_callback(struct netif *n)
+{
+	// Called from core, no need to lock
+	if (!n || !n->state) {
+		return;
+	}
+	VirtualTap *tap = (VirtualTap *)n->state;
+	uint64_t mac = 0;
+	memcpy(&mac, n->hwaddr, n->hwaddr_len);
+	struct zts_netif_details *ifd = new zts_netif_details;
+	ifd->nwid = tap->_nwid;
+	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+	ifd->mac = lwip_htonl(ifd->mac) >> 16;
+	_enqueueEvent(ZTS_EVENT_NETIF_REMOVED, (void*)ifd);
+}
+#endif
+
+/**
+ * Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
+ */
+#if LWIP_NETIF_LINK_CALLBACK
+static void _netif_link_callback(struct netif *n)
+{
+	// Called from core, no need to lock
+	if (!n || !n->state) {
+		return;
+	}
+	VirtualTap *tap = (VirtualTap *)n->state;
+	uint64_t mac = 0;
+	memcpy(&mac, n->hwaddr, n->hwaddr_len);
+	if (n->flags & NETIF_FLAG_LINK_UP) {
+		struct zts_netif_details *ifd = new zts_netif_details;
+		ifd->nwid = tap->_nwid;
+		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+		ifd->mac = lwip_htonl(ifd->mac) >> 16;
+		_enqueueEvent(ZTS_EVENT_NETIF_LINK_UP, (void*)ifd);
+	}
+	if (n->flags & NETIF_FLAG_LINK_UP) {
+		struct zts_netif_details *ifd = new zts_netif_details;
+		ifd->nwid = tap->_nwid;
+		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+		ifd->mac = lwip_htonl(ifd->mac) >> 16;
+		_enqueueEvent(ZTS_EVENT_NETIF_LINK_DOWN, (void*)ifd);
+	}
+}
+#endif
+
+void _lwip_set_callbacks(struct netif *n)
+{
+	if (!n) {
+		return;
+	}
+#if LWIP_NETIF_STATUS_CALLBACK
+	// Not currently used
+	netif_set_status_callback(n, netif_status_callback);
+#endif
+#if LWIP_NETIF_REMOVE_CALLBACK
+	netif_set_remove_callback(n, netif_remove_callback);
+#endif
+#if LWIP_NETIF_LINK_CALLBACK
+	netif_set_link_callback(n, netif_link_callback);
+#endif
+}
+
+static struct zts_netif_details *_lwip_prepare_netif_status_msg(struct netif *n)
+{
+	if (!n || !n->state) {
+		return NULL;
+	}
+	VirtualTap *tap = (VirtualTap*)(n->state);
+	struct zts_netif_details *ifd = new zts_netif_details;
+	ifd->nwid = tap->_nwid;
+	ifd->mtu = n->mtu;
+	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
+	ifd->mac = htonll(ifd->mac) >> 16;
+	return ifd;
+}
+
+static err_t _netif_init4(struct netif *n)
+{
+	if (!n || !n->state) {
+		return ERR_IF;
+	}
+	// Called from netif code, no need to lock
+	n->hwaddr_len = 6;
+	n->name[0]    = '4';
+	n->name[1]    = 'a'+netifCount;
+	n->linkoutput = _lwip_eth_tx;
+	n->output     = etharp_output;
+	n->mtu        = LWIP_MTU < ZT_MAX_MTU ? LWIP_MTU : ZT_MAX_MTU;
+	n->flags      = NETIF_FLAG_BROADCAST
+		| NETIF_FLAG_ETHARP
+		| NETIF_FLAG_ETHERNET
+		| NETIF_FLAG_IGMP
+		| NETIF_FLAG_MLD6
+		| NETIF_FLAG_LINK_UP
+		| NETIF_FLAG_UP;
+	n->hwaddr_len = sizeof(n->hwaddr);
+	VirtualTap *tap = (VirtualTap*)(n->state);
+	tap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
+	return ERR_OK;
+}
+
+static err_t _netif_init6(struct netif *n)
+{
+	if (!n || !n->state) {
+		return ERR_IF;
+	}
+	n->hwaddr_len = sizeof(n->hwaddr);
+	VirtualTap *tap = (VirtualTap*)(n->state);
+	tap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
+	// Called from netif code, no need to lock
+	n->hwaddr_len = 6;
+	n->name[0]    = '6';
+	n->name[1]    = 'a'+netifCount;
+	n->linkoutput = _lwip_eth_tx;
+	n->output_ip6 = ethip6_output;
+	n->mtu        = LWIP_MTU < ZT_MAX_MTU ? LWIP_MTU : ZT_MAX_MTU;
+	n->flags      = NETIF_FLAG_BROADCAST
+	    | NETIF_FLAG_ETHARP
+		| NETIF_FLAG_ETHERNET
+		| NETIF_FLAG_IGMP
+		| NETIF_FLAG_MLD6
+		| NETIF_FLAG_LINK_UP
+		| NETIF_FLAG_UP;
+	return ERR_OK;
+}
+
+void _lwip_init_interface(void *tapref, const InetAddress &ip)
+{
+	char ipbuf[INET6_ADDRSTRLEN];
+	char macbuf[ZTS_MAC_ADDRSTRLEN];
+
+	VirtualTap *vtap = (VirtualTap*)tapref;
+	struct netif *n = NULL;
+	bool isNewNetif = false;
+
+	if (ip.isV4()) {
+		if (vtap->netif4) {
+			n = (struct netif*)vtap->netif4;
+		}
+		else {
+			n = new struct netif;
+			isNewNetif = true;
+			netifCount++;
+		}
+		char nmbuf[INET6_ADDRSTRLEN];
+		static ip4_addr_t ip4, netmask, gw;
+		IP4_ADDR(&gw,127,0,0,1);
+		ip4.addr = *((u32_t *)ip.rawIpData());
+		netmask.addr = *((u32_t *)ip.netmask().rawIpData());
+		LOCK_TCPIP_CORE();
+		netif_add(n, &ip4, &netmask, &gw, (void*)vtap, _netif_init4, tcpip_input);
+		vtap->netif4 = (void*)n;
+		_enqueueEvent(ZTS_EVENT_NETIF_UP, (void*)_lwip_prepare_netif_status_msg(n));
+		UNLOCK_TCPIP_CORE();
+		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
+			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
+		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, nm=%s, tap=%p]",n,
+			macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf), vtap);
+	}
+	if (ip.isV6()) {
+		if (vtap->netif6) {
+			n = (struct netif*)vtap->netif6;
+		}
+		else {
+			n = new struct netif;
+			isNewNetif = true;
+			netifCount++;
+		}
+		static ip6_addr_t ip6;
+		memcpy(&(ip6.addr), ip.rawIpData(), sizeof(ip6.addr));
+		LOCK_TCPIP_CORE();
+		if (isNewNetif) {
+			vtap->netif6 = (void*)n;
+			netif_add(n, NULL, NULL, NULL, (void*)vtap, _netif_init6, ethernet_input);
+			n->ip6_autoconfig_enabled = 1;
+			vtap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
+			netif_create_ip6_linklocal_address(n, 1);
+			netif_set_link_up(n);
+			netif_set_up(n);
+			netif_set_default(n);
+		}
+		netif_add_ip6_address(n,&ip6,NULL);
+		n->output_ip6 = ethip6_output;
+		UNLOCK_TCPIP_CORE();
+		_enqueueEvent(ZTS_EVENT_NETIF_UP, (void*)_lwip_prepare_netif_status_msg(n));
+		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
+			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
+			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
+		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, tap=%p]", n,
+			macbuf, ip.toString(ipbuf), vtap);
+	}
+}
+
+} // namespace ZeroTier

src/VirtualTap.hpp

@@ -14,38 +14,30 @@
 /**
  * @file
  *
- * Virtual Ethernet tap device
+ * Header for virtual ethernet tap device and combined network stack driver
  */
 
-#ifndef LIBZT_VIRTUALTAP_HPP
+#ifndef ZT_VIRTUAL_TAP_HPP
-#define LIBZT_VIRTUALTAP_HPP
+#define ZT_VIRTUAL_TAP_HPP
 
-#ifndef _MSC_VER
+#include "lwip/err.h"
-extern int errno;
-#endif
 
+#define ZTS_LWIP_DRIVER_THREAD_NAME "NetworkStackThread"
+
+#include "MAC.hpp"
 #include "Phy.hpp"
 #include "Thread.hpp"
-#include "InetAddress.hpp"
-#include "MulticastGroup.hpp"
-#include "Mutex.hpp"
-
-#include "Options.h"
-
-#if defined(_WIN32)
-#include <WinSock2.h>
-#include <Windows.h>
-#include <IPHlpApi.h>
-#include <Ifdef.h>
-#endif
 
 namespace ZeroTier {
 
 class Mutex;
+class MAC;
+class MulticastGroup;
+struct InetAddress;
 
 /**
- * A virtual tap device. The ZeroTier core service creates one of these for each
+ * A virtual tap device. The ZeroTier Node Service will create one per
- * virtual network joined. It will be destroyed upon leave().
+ * joined network. It will be destroyed upon leave().
  */
 class VirtualTap
 {
@@ -84,18 +76,18 @@ public:
 	bool hasIpv6Addr();
 
 	/**
-	 * Adds an address to the userspace stack interface associated with this VirtualTap
+	 * Adds an address to the user-space stack interface associated with this VirtualTap
 	 * - Starts VirtualTap main thread ONLY if successful
 	 */
 	bool addIp(const InetAddress &ip);
 
 	/**
-	 * Removes an address from the userspace stack interface associated with this VirtualTap
+	 * Removes an address from the user-space stack interface associated with this VirtualTap
 	 */
 	bool removeIp(const InetAddress &ip);
 
 	/**
-	 * Presents data to the userspace stack
+	 * Presents data to the user-space stack
 	 */
 	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,
 		unsigned int len);
@@ -105,11 +97,6 @@ public:
 	 */
 	std::string deviceName() const;
 
-	/**
-	 * Get Node ID (ZT address)
-	 */
-	std::string nodeId() const;
-
 	/**
 	 * Set friendly name
 	 */
@@ -152,10 +139,6 @@ public:
 	void (*_handler)(void *, void *, uint64_t, const MAC &, const MAC &, unsigned int, unsigned int,
 		const void *, unsigned int);
 
-	//////////////////////////////////////////////////////////////////////////////
-	// Lower-level lwIP netif handling and traffic handling readiness           //
-	//////////////////////////////////////////////////////////////////////////////
-
 	void *netif4 = NULL;
 	void *netif6 = NULL;
 
@@ -164,20 +147,10 @@ public:
 	 */
 	uint64_t _lastConfigUpdateTime = 0;
 
-	/**
-	 * The last time that a callback notification was sent to the user application signalling
-	 * that this interface is ready to process traffic.
-	 */
-	uint64_t _lastReadyReportTime = 0;
-
 	void lastConfigUpdate(uint64_t lastConfigUpdateTime);
 
 	int _networkStatus = 0;
 
-	//////////////////////////////////////////////////////////////////////////////
-	// Vars                                                                     //
-	//////////////////////////////////////////////////////////////////////////////
-
 	std::vector<std::pair<InetAddress, InetAddress> > routes;
 
 	char vtap_full_name[64];
@@ -205,17 +178,6 @@ public:
 	std::vector<MulticastGroup> _multicastGroups;
 	Mutex _multicastGroups_m;
 
-	/*
-	 * Timestamp of last run of housekeeping
-	 * SEE: ZT_HOUSEKEEPING_INTERVAL in ZeroTier.h
-	 */
-	uint64_t last_housekeeping_ts = 0;
-
-	/**
-	 * Performs miscellaneous background tasks
-	 */
-	void Housekeeping();
-
 	//////////////////////////////////////////////////////////////////////////////
 	// Not used in this implementation                                          //
 	//////////////////////////////////////////////////////////////////////////////
@@ -228,8 +190,135 @@ public:
 	void phyOnTcpClose(PhySocket *sock,void **uptr);
 	void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len);
 	void phyOnTcpWritable(PhySocket *sock,void **uptr);
+	void phyOnUnixClose(PhySocket *sock,void **uptr);
 };
 
+/**
+ * @brief Return whether a given netif's NETIF_FLAG_UP flag is set
+ *
+ * @usage This is a convenience function to encapsulate a macro
+ */
+bool _lwip_is_netif_up(void *netif);
+
+/**
+ * @brief Increase the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when we know the stack won't be used by any virtual taps
+ */
+void _lwip_hibernate_driver();
+
+/**
+ * @brief Decrease the delay multiplier for the main driver loop
+ *
+ * @usage This should be called when at least one virtual tap is active
+ */
+void _lwip_wake_driver();
+
+/**
+ * Returns whether the lwIP network stack is up and ready to process traffic
+ */
+bool _lwip_is_up();
+
+/**
+ * @brief Initialize network stack semaphores, threads, and timers.
+ *
+ * @usage This is called during the initial setup of each VirtualTap but is only allowed to execute once
+ * @return
+ */
+void _lwip_driver_init();
+
+/**
+ * @brief Shutdown the stack as completely as possible (not officially supported by lwIP)
+ *
+ * @usage This is to be called after it is determined that no further network activity will take place.
+ * The tcpip thread will be stopped, all interfaces will be brought down and all resources will
+ * be deallocated. A full application restart will be required to bring the stack back online.
+ * @return
+ */
+void _lwip_driver_shutdown();
+
+/**
+ * @brief Requests that a netif be brought down and removed.
+ *
+ * @return
+ */
+void _lwip_remove_netif(void *netif);
+
+/**
+ * @brief Initialize and start the DNS client
+ *
+ * @usage Called after lwip_driver_init()
+ * @return
+ */
+void _lwip_dns_init();
+
+/**
+ * @brief Starts DHCP timers
+ *
+ * @usage lwip_driver_init()
+ * @return
+ */
+void _lwip_start_dhcp(void *netif);
+
+/**
+ * @brief Called when the status of a netif changes:
+ *  - Interface is up/down (ZTS_EVENT_NETIF_UP, ZTS_EVENT_NETIF_DOWN)
+ *  - Address changes while up (ZTS_EVENT_NETIF_NEW_ADDRESS)
+ */
+#if LWIP_NETIF_STATUS_CALLBACK
+static void _netif_status_callback(struct netif *netif);
+#endif
+
+/**
+ * @brief Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
+ */
+#if LWIP_NETIF_REMOVE_CALLBACK
+static void _netif_remove_callback(struct netif *netif);
+#endif
+
+/**
+ * @brief Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
+ */
+#if LWIP_NETIF_LINK_CALLBACK
+static void _netif_link_callback(struct netif *netif);
+#endif
+
+/**
+ * @brief Set up an interface in the network stack for the VirtualTap.
+ *
+ * @param
+ * @param tapref Reference to VirtualTap that will be responsible for sending and receiving data
+ * @param ip Virtual IP address for this ZeroTier VirtualTap interface
+ * @return
+ */
+void _lwip_init_interface(void *tapref, const InetAddress &ip);
+
+/**
+ * @brief Called from the stack, outbound ethernet frames from the network stack enter the ZeroTier virtual wire here.
+ *
+ * @usage This shall only be called from the stack or the stack driver. Not the application thread.
+ * @param netif Transmits an outgoing Ethernet fram from the network stack onto the ZeroTier virtual wire
+ * @param p A pointer to the beginning of a chain pf struct pbufs
+ * @return
+ */
+err_t _lwip_eth_tx(struct netif *netif, struct pbuf *p);
+
+/**
+ * @brief Receives incoming Ethernet frames from the ZeroTier virtual wire
+ *
+ * @usage This shall be called from the VirtualTap's I/O thread (via VirtualTap::put())
+ * @param tap Pointer to VirtualTap from which this data comes
+ * @param from Origin address (virtual ZeroTier hardware address)
+ * @param to Intended destination address (virtual ZeroTier hardware address)
+ * @param etherType Protocol type
+ * @param data Pointer to Ethernet frame
+ * @param len Length of Ethernet frame
+ * @return
+ */
+void _lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
+	const void *data, unsigned int len);
+
 } // namespace ZeroTier
 
 #endif // _H
+

src/lwipDriver.cpp

@@ -1,535 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-/**
- * @file
- *
- * lwIP network stack driver
- */
-
-#include <queue>
-
-#include "MAC.hpp"
-#include "Mutex.hpp"
-
-#include "netif/ethernet.h"
-#include "lwip/netif.h"
-#include "lwip/etharp.h"
-#include "lwip/tcpip.h"
-#include "lwip/mem.h"
-#include "lwip/memp.h"
-#include "lwip/sys.h"
-#include "lwip/tcp.h"
-#include "lwip/timeouts.h"
-#include "lwip/stats.h"
-#include "lwip/ethip6.h"
-#include "lwip/ip_addr.h"
-#include "lwip/nd6.h"
-#include "lwip/netifapi.h"
-
-#ifdef LWIP_STATS
-#include "lwip/stats.h"
-#endif
-
-#include "VirtualTap.hpp"
-#include "lwipDriver.hpp"
-#include "ZeroTier.h"
-#include "Controls.hpp"
-
-extern void postEvent(uint64_t eventCode, void *arg);
-extern void postEvent(uint64_t eventCode);
-
-#if defined(_WIN32)
-#include <time.h>
-#endif
-
-/**
- * Length of human-readable MAC address string
- */
-#define ZTS_MAC_ADDRSTRLEN 18
-
-#ifndef htonll
-#define htonll(x) ((1==htonl(1)) ? (x) : ((uint64_t)htonl((x) & 0xFFFFFFFF) << 32) | htonl((x) >> 32))
-#endif
-
-namespace ZeroTier {
-
-bool _has_exited = false;
-int hibernationDelayMultiplier = 1;
-int netifCount = 0;
-extern bool _run_lwip_tcpip;
-Mutex lwip_driver_m;
-
-void lwip_sleep(long ms)
-{
-#if defined(_WIN32)
-	Sleep(ms*hibernationDelayMultiplier);
-#else
-	usleep(ms*1000*hibernationDelayMultiplier);
-#endif
-}
-
-void lwip_hibernate_driver()
-{
-	hibernationDelayMultiplier = ZTS_HIBERNATION_MULTIPLIER;
-}
-
-void lwip_wake_driver()
-{
-	hibernationDelayMultiplier = 1;
-}
-
-// Callback for when the TCPIP thread has been successfully started
-static void tcpip_init_done(void *arg)
-{
-	sys_sem_t *sem;
-	sem = (sys_sem_t *)arg;
-	_run_lwip_tcpip = true;
-	postEvent(ZTS_EVENT_STACK_UP);
-	sys_sem_signal(sem);
-}
-
-static void main_lwip_driver_loop(void *arg)
-{
-#if defined(__linux__)
-	pthread_setname_np(pthread_self(), ZTS_LWIP_DRIVER_THREAD_NAME);
-#endif
-#if defined(__APPLE__)
-	pthread_setname_np(ZTS_LWIP_DRIVER_THREAD_NAME);
-#endif
-	sys_sem_t sem;
-	LWIP_UNUSED_ARG(arg);
-	if (sys_sem_new(&sem, 0) != ERR_OK) {
-		DEBUG_ERROR("failed to create semaphore");
-	}
-	tcpip_init(tcpip_init_done, &sem);
-	sys_sem_wait(&sem);
-	// Main loop
-	while(_run_lwip_tcpip) {
-		lwip_sleep(LWIP_DRIVER_LOOP_INTERVAL);
-	}
-	_has_exited = true;
-	postEvent(ZTS_EVENT_STACK_DOWN);
-}
-
-bool lwip_is_up()
-{
-	Mutex::Lock _l(lwip_driver_m);
-	return _run_lwip_tcpip;
-}
-
-bool lwip_has_previously_shutdown()
-{
-	Mutex::Lock _l(lwip_driver_m);
-	return _has_exited;
-}
-
-void lwip_driver_init()
-{
-	if (lwip_is_up()) {
-		return;
-	}
-	if (lwip_has_previously_shutdown()) {
-		return;
-	}
-	Mutex::Lock _l(lwip_driver_m);
-#if defined(_WIN32)
-	sys_init(); // Required for win32 init of critical sections
-#endif
-	sys_thread_new(ZTS_LWIP_DRIVER_THREAD_NAME, main_lwip_driver_loop,
-		NULL, DEFAULT_THREAD_STACKSIZE, DEFAULT_THREAD_PRIO);
-}
-
-void lwip_driver_shutdown()
-{
-	if (lwip_has_previously_shutdown()) {
-		return;
-	}
-	Mutex::Lock _l(lwip_driver_m);
-	// Set flag to stop sending frames into the core
-	_run_lwip_tcpip = false;
-	// Wait until the main lwIP thread has exited
-	while (!_has_exited) { lwip_sleep(LWIP_DRIVER_LOOP_INTERVAL); }
-	/*
-	if (tcpip_shutdown() == ERR_OK) {
-		sys_timeouts_free();
-	}
-	*/
-}
-
-void lwip_remove_netif(void *netif)
-{
-	if (!netif) {
-		return;
-	}
-	struct netif *n = (struct netif*)netif;
-	LOCK_TCPIP_CORE();
-	netif_remove(n);
-	netif_set_down(n);
-	netif_set_link_down(n);
-	UNLOCK_TCPIP_CORE();
-}
-
-err_t lwip_eth_tx(struct netif *n, struct pbuf *p)
-{
-	if (!n) {
-		return ERR_IF;
-	}
-	struct pbuf *q;
-	char buf[ZT_MAX_MTU+32];
-	char *bufptr;
-	int totalLength = 0;
-
-	VirtualTap *tap = (VirtualTap*)n->state;
-	bufptr = buf;
-	for (q = p; q != NULL; q = q->next) {
-		memcpy(bufptr, q->payload, q->len);
-		bufptr += q->len;
-		totalLength += q->len;
-	}
-	struct eth_hdr *ethhdr;
-	ethhdr = (struct eth_hdr *)buf;
-
-	MAC src_mac;
-	MAC dest_mac;
-	src_mac.setTo(ethhdr->src.addr, 6);
-	dest_mac.setTo(ethhdr->dest.addr, 6);
-
-	char *data = buf + sizeof(struct eth_hdr);
-	int len = totalLength - sizeof(struct eth_hdr);
-	int proto = Utils::ntoh((uint16_t)ethhdr->type);
-	tap->_handler(tap->_arg, NULL, tap->_nwid, src_mac, dest_mac, proto, 0, data, len);
-	if (ZT_MSG_TRANSFER == true) {
-		char flagbuf[32];
-		memset(&flagbuf, 0, 32);
-		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
-		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-			ethhdr->dest.addr[0], ethhdr->dest.addr[1], ethhdr->dest.addr[2], 
-			ethhdr->dest.addr[3], ethhdr->dest.addr[4], ethhdr->dest.addr[5]);
-		MAC mac;
-		mac.setTo(ethhdr->dest.addr, 6);
-		mac.toAddress(tap->_nwid).toString(nodeBuf);
-		/*
-		DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] ethertype=0x%04x %s", totalLength, macBuf, nodeBuf, tap->nodeId().c_str(),
-			Utils::ntoh(ethhdr->type), flagbuf);
-		*/
-	}
-	return ERR_OK;
-}
-
-void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
-	const void *data, unsigned int len)
-{
-#ifdef LWIP_STATS
-	stats_display();
-#endif
-	if (!_run_lwip_tcpip) {
-		return;
-	}
-	struct pbuf *p,*q;
-	struct eth_hdr ethhdr;
-	from.copyTo(ethhdr.src.addr, 6);
-	to.copyTo(ethhdr.dest.addr, 6);
-	ethhdr.type = Utils::hton((uint16_t)etherType);
-
-	if (ZT_MSG_TRANSFER == true) {
-		char flagbuf[32];
-		memset(&flagbuf, 0, 32);
-		char macBuf[ZTS_MAC_ADDRSTRLEN], nodeBuf[16];
-		snprintf(macBuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-			ethhdr.dest.addr[0], ethhdr.dest.addr[1], ethhdr.dest.addr[2],
-			ethhdr.dest.addr[3], ethhdr.dest.addr[4], ethhdr.dest.addr[5]);
-		MAC mac;
-		mac.setTo(ethhdr.src.addr, 6);
-		mac.toAddress(tap->_nwid).toString(nodeBuf);
-		/*
-		DEBUG_TRANS("len=%5d dst=%s [%s RX --> %s] ethertype=0x%04x %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
-			Utils::ntoh(ethhdr.type), flagbuf);
-		*/
-	}
-
-	p = pbuf_alloc(PBUF_RAW, (uint16_t)len+sizeof(struct eth_hdr), PBUF_RAM);
-	if (!p) {
-		DEBUG_ERROR("dropped packet: unable to allocate memory for pbuf");
-		return;
-	}
-	// First pbuf gets ethernet header at start
-	q = p;
-	if (q->len < sizeof(ethhdr)) {
-		pbuf_free(p);
-		p = NULL;
-		DEBUG_ERROR("dropped packet: first pbuf smaller than ethernet header");
-		return;
-	}
-	// Copy frame data into pbuf
-	const char *dataptr = reinterpret_cast<const char *>(data);
-	memcpy(q->payload,&ethhdr,sizeof(ethhdr));
-	int remainingPayloadSpace = q->len - sizeof(ethhdr);
-	memcpy((char*)q->payload + sizeof(ethhdr),dataptr,remainingPayloadSpace);
-	dataptr += remainingPayloadSpace;
-	// Remaining pbufs (if any) get rest of data
-	while ((q = q->next)) {
-		memcpy(q->payload,dataptr,q->len);
-		dataptr += q->len;
-	}
-	// Feed packet into stack
-	int err;
-
-	if (Utils::ntoh(ethhdr.type) == 0x800 || Utils::ntoh(ethhdr.type) == 0x806) {
-		if ((err = ((struct netif *)tap->netif4)->input(p, (struct netif *)tap->netif4)) != ERR_OK) {
-			DEBUG_ERROR("packet input error (%d)", err);
-			pbuf_free(p);
-		}
-	}
-	if (Utils::ntoh(ethhdr.type) == 0x86DD) {
-		if ((err = ((struct netif *)tap->netif6)->input(p, (struct netif *)tap->netif6)) != ERR_OK) {
-			DEBUG_ERROR("packet input error (%d)", err);
-			pbuf_free(p);
-		}
-	}
-}
-
-/*
-static void print_netif_info(struct netif *n) {
-	DEBUG_INFO("n=%p, %c%c, %d, o=%p, o6=%p, mc=%x:%x:%x:%x:%x:%x, hwln=%d, st=%p, flgs=%d\n",
-		n,
-		n->name[0],
-		n->name[1],
-		n->mtu,
-		n->output,
-		n->output_ip6,
-		n->hwaddr[0],
-		n->hwaddr[1],
-		n->hwaddr[2],
-		n->hwaddr[3],
-		n->hwaddr[4],
-		n->hwaddr[5],
-		n->hwaddr_len,
-		n->state,
-		n->flags
-	);
-}
-*/
-
-bool lwip_is_netif_up(void *n)
-{
-	if (!n) {
-		return false;
-	}
-	LOCK_TCPIP_CORE();
-	bool result = netif_is_up((struct netif*)n);
-	UNLOCK_TCPIP_CORE();
-	return result;
-}
-
-/**
- * Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
- */
-#if LWIP_NETIF_REMOVE_CALLBACK
-static void netif_remove_callback(struct netif *n)
-{
-	// Called from core, no need to lock
-	if (!n || !n->state) {
-		return;
-	}
-	VirtualTap *tap = (VirtualTap *)n->state;
-	uint64_t mac = 0;
-	memcpy(&mac, n->hwaddr, n->hwaddr_len);
-	struct zts_netif_details *ifd = new zts_netif_details;
-	ifd->nwid = tap->_nwid;
-	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
-	ifd->mac = lwip_htonl(ifd->mac) >> 16;
-	postEvent(ZTS_EVENT_NETIF_REMOVED, (void*)ifd);
-}
-#endif
-
-/**
- * Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
- */
-#if LWIP_NETIF_LINK_CALLBACK
-static void netif_link_callback(struct netif *n)
-{
-	// Called from core, no need to lock
-	if (!n || !n->state) {
-		return;
-	}
-	VirtualTap *tap = (VirtualTap *)n->state;
-	uint64_t mac = 0;
-	memcpy(&mac, n->hwaddr, n->hwaddr_len);
-	if (n->flags & NETIF_FLAG_LINK_UP) {
-		struct zts_netif_details *ifd = new zts_netif_details;
-		ifd->nwid = tap->_nwid;
-		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
-		ifd->mac = lwip_htonl(ifd->mac) >> 16;
-		postEvent(ZTS_EVENT_NETIF_LINK_UP, (void*)ifd);
-	}
-	if (n->flags & NETIF_FLAG_LINK_UP) {
-		struct zts_netif_details *ifd = new zts_netif_details;
-		ifd->nwid = tap->_nwid;
-		memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
-		ifd->mac = lwip_htonl(ifd->mac) >> 16;
-		postEvent(ZTS_EVENT_NETIF_LINK_DOWN, (void*)ifd);
-	}
-}
-#endif
-
-void lwip_set_callbacks(struct netif *n)
-{
-	if (!n) {
-		return;
-	}
-#if LWIP_NETIF_STATUS_CALLBACK
-	// Not currently used
-	netif_set_status_callback(n, netif_status_callback);
-#endif
-#if LWIP_NETIF_REMOVE_CALLBACK
-	netif_set_remove_callback(n, netif_remove_callback);
-#endif
-#if LWIP_NETIF_LINK_CALLBACK
-	netif_set_link_callback(n, netif_link_callback);
-#endif
-}
-
-static struct zts_netif_details *lwip_prepare_netif_status_msg(struct netif *n)
-{
-	if (!n || !n->state) {
-		return NULL;
-	}
-	VirtualTap *tap = (VirtualTap*)(n->state);
-	struct zts_netif_details *ifd = new zts_netif_details;
-	ifd->nwid = tap->_nwid;
-	ifd->mtu = n->mtu;
-	memcpy(&(ifd->mac), n->hwaddr, n->hwaddr_len);
-	ifd->mac = htonll(ifd->mac) >> 16;
-	return ifd;
-}
-
-static err_t netif_init(struct netif *n)
-{
-	if (!n || !n->state) {
-		return ERR_IF;
-	}
-	// Called from netif code, no need to lock
-	n->hwaddr_len = 6;
-	n->name[0]    = '4';
-	n->name[1]    = 'a'+netifCount;
-	n->linkoutput = lwip_eth_tx;
-	n->output     = etharp_output;
-	n->mtu        = LWIP_MTU < ZT_MAX_MTU ? LWIP_MTU : ZT_MAX_MTU;
-	n->flags      = NETIF_FLAG_BROADCAST
-		| NETIF_FLAG_ETHARP
-		| NETIF_FLAG_ETHERNET
-		| NETIF_FLAG_IGMP
-		| NETIF_FLAG_MLD6
-		| NETIF_FLAG_LINK_UP
-		| NETIF_FLAG_UP;
-	n->hwaddr_len = sizeof(n->hwaddr);
-	VirtualTap *tap = (VirtualTap*)(n->state);
-	tap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
-	return ERR_OK;
-}
-
-static err_t netif_init6(struct netif *n)
-{
-	if (!n || !n->state) {
-		return ERR_IF;
-	}
-	n->hwaddr_len = sizeof(n->hwaddr);
-	VirtualTap *tap = (VirtualTap*)(n->state);
-	tap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
-	// Called from netif code, no need to lock
-	n->hwaddr_len = 6;
-	n->name[0]    = '6';
-	n->name[1]    = 'a'+netifCount;
-	n->linkoutput = lwip_eth_tx;
-	n->output_ip6 = ethip6_output;
-	n->mtu        = LWIP_MTU < ZT_MAX_MTU ? LWIP_MTU : ZT_MAX_MTU;
-	n->flags      = NETIF_FLAG_BROADCAST
-	    | NETIF_FLAG_ETHARP
-		| NETIF_FLAG_ETHERNET
-		| NETIF_FLAG_IGMP
-		| NETIF_FLAG_MLD6
-		| NETIF_FLAG_LINK_UP
-		| NETIF_FLAG_UP;
-	return ERR_OK;
-}
-
-void lwip_init_interface(void *tapref, const InetAddress &ip)
-{
-	char ipbuf[INET6_ADDRSTRLEN];
-	char macbuf[ZTS_MAC_ADDRSTRLEN];
-
-	VirtualTap *vtap = (VirtualTap*)tapref;
-	struct netif *n = NULL;
-	bool isNewNetif = false;
-
-	if (ip.isV4()) {
-		if (vtap->netif4) {
-			n = (struct netif*)vtap->netif4;
-		}
-		else {
-			n = new struct netif;
-			isNewNetif = true;
-			netifCount++;
-		}
-		char nmbuf[INET6_ADDRSTRLEN];
-		static ip4_addr_t ip4, netmask, gw;
-		IP4_ADDR(&gw,127,0,0,1);
-		ip4.addr = *((u32_t *)ip.rawIpData());
-		netmask.addr = *((u32_t *)ip.netmask().rawIpData());
-		LOCK_TCPIP_CORE();
-		netif_add(n, &ip4, &netmask, &gw, (void*)vtap, netif_init, tcpip_input);
-		vtap->netif4 = (void*)n;
-		postEvent(ZTS_EVENT_NETIF_UP, (void*)lwip_prepare_netif_status_msg(n));
-		UNLOCK_TCPIP_CORE();
-		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
-			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
-		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, nm=%s, tap=%p]",n,
-			macbuf, ip.toString(ipbuf), ip.netmask().toString(nmbuf), vtap);
-	}
-	if (ip.isV6()) {
-		if (vtap->netif6) {
-			n = (struct netif*)vtap->netif6;
-		}
-		else {
-			n = new struct netif;
-			isNewNetif = true;
-			netifCount++;
-		}
-		static ip6_addr_t ip6;
-		memcpy(&(ip6.addr), ip.rawIpData(), sizeof(ip6.addr));
-		LOCK_TCPIP_CORE();
-		if (isNewNetif) {
-			vtap->netif6 = (void*)n;
-			netif_add(n, NULL, NULL, NULL, (void*)vtap, netif_init6, ethernet_input);
-			n->ip6_autoconfig_enabled = 1;
-			vtap->_mac.copyTo(n->hwaddr, n->hwaddr_len);
-			netif_create_ip6_linklocal_address(n, 1);
-			netif_set_link_up(n);
-			netif_set_up(n);
-			netif_set_default(n);
-		}
-		netif_add_ip6_address(n,&ip6,NULL);
-		n->output_ip6 = ethip6_output;
-		UNLOCK_TCPIP_CORE();
-		postEvent(ZTS_EVENT_NETIF_UP, (void*)lwip_prepare_netif_status_msg(n));
-		snprintf(macbuf, ZTS_MAC_ADDRSTRLEN, "%02x:%02x:%02x:%02x:%02x:%02x",
-			n->hwaddr[0], n->hwaddr[1], n->hwaddr[2],
-			n->hwaddr[3], n->hwaddr[4], n->hwaddr[5]);
-		DEBUG_INFO("initialized netif=%p as [mac=%s, addr=%s, tap=%p]", n,
-			macbuf, ip.toString(ipbuf), vtap);
-	}
-}
-
-} // namespace ZeroTier

src/lwipDriver.hpp

@@ -1,173 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-/**
- * @file
- *
- * lwIP network stack driver
- */
-
-#ifndef LIBZT_LWIP_DRIVER_HPP
-#define LIBZT_LWIP_DRIVER_HPP
-
-#include "Debug.hpp"
-#include "lwip/err.h"
-
-namespace ZeroTier {
-
-class MAC;
-class Mutex;
-class VirtualTap;
-struct InetAddress;
-
-/**
- * @brief Structure used to associate packets with interfaces.
- */
-struct zts_sorted_packet
-{
-	// lwIP pbuf containing packet (originally encapsulated by ZT packet)
-	struct pbuf *p;
-	// ZT VirtualTap from which this packet originates
-	VirtualTap *vtap;
-	// lwIP netif we should accept this packet on
-	struct netif *n;
-};
-
-/**
- * @brief Return whether a given netif's NETIF_FLAG_UP flag is set
- *
- * @usage This is a convenience function to encapsulate a macro
- */
-bool lwip_is_netif_up(void *netif);
-
-/**
- * @brief Increase the delay multiplier for the main driver loop
- *
- * @usage This should be called when we know the stack won't be used by any virtual taps
- */
-void lwip_hibernate_driver();
-
-/**
- * @brief Decrease the delay multiplier for the main driver loop
- *
- * @usage This should be called when at least one virtual tap is active
- */
-void lwip_wake_driver();
-
-/**
- * Returns whether the lwIP network stack is up and ready to process traffic
- */
-bool lwip_is_up();
-
-/**
- * @brief Initialize network stack semaphores, threads, and timers.
- *
- * @usage This is called during the initial setup of each VirtualTap but is only allowed to execute once
- * @return
- */
-void lwip_driver_init();
-
-/**
- * @brief Shutdown the stack as completely as possible (not officially supported by lwIP)
- *
- * @usage This is to be called after it is determined that no further network activity will take place.
- * The tcpip thread will be stopped, all interfaces will be brought down and all resources will 
- * be deallocated. A full application restart will be required to bring the stack back online.
- * @return
- */
-void lwip_driver_shutdown();
-
-/**
- * @brief Requests that a netif be brought down and removed.
- *
- * @return
- */
-void lwip_remove_netif(void *netif);
-
-/**
- * @brief Initialize and start the DNS client
- *
- * @usage Called after lwip_driver_init()
- * @return
- */
-void lwip_dns_init();
-
-/**
- * @brief Starts DHCP timers
- *
- * @usage lwip_driver_init()
- * @return
- */
-void lwip_start_dhcp(void *netif);
-
-/**
- * @brief Called when the status of a netif changes:
- *  - Interface is up/down (ZTS_EVENT_NETIF_UP, ZTS_EVENT_NETIF_DOWN)
- *  - Address changes while up (ZTS_EVENT_NETIF_NEW_ADDRESS)
- */
-#if LWIP_NETIF_STATUS_CALLBACK
-static void netif_status_callback(struct netif *netif);
-#endif
-
-/**
- * @brief Called when a netif is removed (ZTS_EVENT_NETIF_INTERFACE_REMOVED)
- */
-#if LWIP_NETIF_REMOVE_CALLBACK
-static void netif_remove_callback(struct netif *netif);
-#endif
-
-/**
- * @brief Called when a link is brought up or down (ZTS_EVENT_NETIF_LINK_UP, ZTS_EVENT_NETIF_LINK_DOWN)
- */
-#if LWIP_NETIF_LINK_CALLBACK
-static void netif_link_callback(struct netif *netif);
-#endif
-
-/**
- * @brief Set up an interface in the network stack for the VirtualTap.
- *
- * @param
- * @param tapref Reference to VirtualTap that will be responsible for sending and receiving data
- * @param ip Virtual IP address for this ZeroTier VirtualTap interface
- * @return
- */
-void lwip_init_interface(void *tapref, const InetAddress &ip);
-
-/**
- * @brief Called from the stack, outbound ethernet frames from the network stack enter the ZeroTier virtual wire here.
- *
- * @usage This shall only be called from the stack or the stack driver. Not the application thread.
- * @param netif Transmits an outgoing Ethernet fram from the network stack onto the ZeroTier virtual wire
- * @param p A pointer to the beginning of a chain pf struct pbufs
- * @return
- */
-err_t lwip_eth_tx(struct netif *netif, struct pbuf *p);
-
-/**
- * @brief Receives incoming Ethernet frames from the ZeroTier virtual wire
- *
- * @usage This shall be called from the VirtualTap's I/O thread (via VirtualTap::put())
- * @param tap Pointer to VirtualTap from which this data comes
- * @param from Origin address (virtual ZeroTier hardware address)
- * @param to Intended destination address (virtual ZeroTier hardware address)
- * @param etherType Protocol type
- * @param data Pointer to Ethernet frame
- * @param len Length of Ethernet frame
- * @return
- */
-void lwip_eth_rx(VirtualTap *tap, const MAC &from, const MAC &to, unsigned int etherType,
-	const void *data, unsigned int len);
-
-} // namespace ZeroTier
-
-#endif // _H