Added differentiation of LIBZT_DEBUG and NS_DEBUG flags, other minor tweaks

FAQ.md

@@ -5,13 +5,13 @@
 
 Yes! - Just let us know, and we will work out a licensing scheme. You will need to build the library with the `lwIP` network stack. In the case that you're a non-profit, or are developing open source software, you can use this entirely for free and may choose either `picoTCP` or `lwIP` as your network stack.
 
-### Application or service won't fully come online
+### My application or service won't fully come online
 
 In rare circumstances it can take a substantial amount of time for a libzt instance to come online and become reachable. In cases where it never seems to progress beyond this stage you should check to make sure there are no rogue processes on the machines using the same ZeroTier identity files, or no other instances on your ZeroTier network using said identity files. If this doesn't help. Contact us.
 
 ### How do I get debug output?
 
-In order of relevance, enable the following: For `libzt`-specific debug output which basically includes the POSIX socket emulation layer and network stack *drivers*, enable `LIBZT_DEBUG=1`. For situations where you suspect that the problem might lie within the network stack itself, enable `NS_DEBUG=1`. You should also check out `include/libzt.h` for additional fine-grained debug options for each network stack. If you think that your problem is with the ZeroTier protocol itself, enable `ZT_DEBUG=1`. This will output what's happening in the ZeroTier core. Note, you can enable all of these at once if you're brave. Additionally, you can use the `LIBZT_SANITIZE=1` flag to build against the [AddressSanitization]() library.
+In order of relevance, enable the following: For `libzt`-specific debug output which basically includes the POSIX socket emulation layer and network stack *drivers*, enable `LIBZT_DEBUG=1`. For situations where you suspect that the problem might lie within the network stack itself, enable `NS_DEBUG=1`. You should also check out `include/libzt.h` for additional fine-grained debug options for each network stack. If you think that your problem is with the ZeroTier protocol itself, enable `ZT_DEBUG=1`. This will output what's happening in the ZeroTier core. Note, you can enable all of these at once if you're brave. Additionally, you can use the `LIBZT_SANITIZE=1` flag to build against the [AddressSanitization](https://github.com/google/sanitizers/wiki/AddressSanitizer) library.
 
 ### Versioning
 

README.md

@@ -19,11 +19,11 @@ Pre-Built Binaries Here: [zerotier.com/download.shtml](https://zerotier.com/down
 ```
 #include "libzt.h"
 
-char *str = "welcome to the machine"; // test msg 
+char *str = "welcome to the machine";
 char *nwid = "c7cd7c9e1b0f52a2";      // network to join
 char *path = "zt1";                   // path where this node's keys and configs will be stored
-char *ip = "10.8.8.42";               // host on ZeroTier network
+char *ip = "10.8.8.42";               // remote address
-int port = 8080;                      // resource's port
+int port = 8080;                      // remote port
 
 struct sockaddr_in addr;
 addr.sin_family = AF_INET;
@@ -31,10 +31,26 @@ addr.sin_addr.s_addr = inet_addr(ip);
 addr.sin_port = hton(port);	
 
 zts_simple_start(path, nwid);
-int fd = zts_socket(AF_INET, SOCK_STREAM, 0);
+
-zts_connect(fd, (const struct sockaddr *)addr, sizeof(addr));
+int fd, err = 0;
-zts_write(fd, str, strlen(str));
+if ((fd = zts_socket(AF_INET, SOCK_STREAM, 0)) < 0) {
-zts_close(fd);
+	printf("error creating socket\n");
+	return -1;
+}
+if ((err = zts_connect(fd, (const struct sockaddr *)addr, sizeof(addr))) < 0) {
+	printf("error connecting to remote host\n");
+	return -1;
+}
+if ((err = zts_write(fd, str, strlen(str))) < 0) {
+	printf("error writing to socket\n");
+	return -1;
+}
+if ((err = zts_close(fd)) < 0) {
+	printf("error closing socket\n");
+	return -1;
+}
+
+zts_stop();
 ```
 
 Bindings for various [languages](examples)

include/Debug.hpp

@@ -73,82 +73,104 @@
 		#define ZT_LOG_TAG "ZTSDK"
 #endif
 
-#define DEBUG_LWIP(fmt, args...) fprintf(stderr, ZT_CYN "LWIP : %17s:%5d:%25s: " fmt   \
+// Network stack debugging
-	ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+#if defined(NS_DEBUG)
+	#define DEBUG_LWIP(fmt, args...) fprintf(stderr, ZT_YEL "LWIP : %17s:%5d:%25s: " fmt   \
+		ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
 
-#if ZT_DEBUG_LEVEL >= ZT_MSG_TEST
+	#define DEBUG_STACK(fmt, args...) fprintf(stderr, ZT_YEL "STACK: %17s:%5d:%25s: " fmt   \
-	#define DEBUG_TEST(fmt, args...) fprintf(stderr, ZT_CYN "TEST : %17s:%5d:%25s: " fmt   \
+		ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		"\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
 #else
-	#define DEBUG_ERROR(fmt, args...)
+	#define DEBUG_LWIP(fmt, args...)
-#endif
-
-#if ZT_DEBUG_LEVEL >= ZT_MSG_ERROR
-	#define DEBUG_ERROR(fmt, args...) fprintf(stderr, ZT_RED "ERROR: %17s:%5d:%25s: " fmt   \
-		"\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-#else
-	#define DEBUG_ERROR(fmt, args...)
-#endif
- 
-#if ZT_DEBUG_LEVEL >= ZT_MSG_INFO
-	#if defined(__ANDROID__)
-		#define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
-			"INFO : %17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-		#define DEBUG_BLANK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
-			"INFO : %17s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-		#define DEBUG_ATTN(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
-			"INFO : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-		#define DEBUG_STACK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
-			"STACK: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-	#else
-		#define DEBUG_INFO(fmt, args...) fprintf(stderr,                                               \
-			"INFO : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		#define DEBUG_ATTN(fmt, args...) fprintf(stderr, ZT_CYN                                        \
-			"ATTN : %17s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		#define DEBUG_STACK(fmt, args...) fprintf(stderr, ZT_YEL                                       \
-			"STACK: %17s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-		#define DEBUG_BLANK(fmt, args...) fprintf(stderr,                                              \
-			"INFO : %17s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, ##args)
-#endif
-#else
-	#define DEBUG_INFO(fmt, args...)
-	#define DEBUG_BLANK(fmt, args...)
-	#define DEBUG_ATTN(fmt, args...)
 	#define DEBUG_STACK(fmt, args...)
 #endif
 
-#if ZT_DEBUG_LEVEL >= ZT_MSG_TRANSFER
+// libzt POSIX socket emulation layer debugging
-	#if defined(__ANDROID__)
+#if defined(LIBZT_DEBUG)
-		#define DEBUG_TRANS(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_TEST
-			"TRANS: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#define DEBUG_TEST(fmt, args...) fprintf(stderr, ZT_CYN "TEST : %17s:%5d:%25s: " fmt   \
-	#else
-		#define DEBUG_TRANS(fmt, args...) fprintf(stderr, ZT_GRN "TRANS: %17s:%5d:%25s: " fmt \
 			"\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+	#else
+		#define DEBUG_ERROR(fmt, args...)
 	#endif
-#else
+
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_ERROR
+		#define DEBUG_ERROR(fmt, args...) fprintf(stderr, ZT_RED "ERROR: %17s:%5d:%25s: " fmt   \
+			"\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+	#else
+		#define DEBUG_ERROR(fmt, args...)
+	#endif
+	 
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_INFO
+		#if defined(__ANDROID__)
+			#define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
+				"INFO : %17s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+			#define DEBUG_BLANK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
+				"INFO : %17s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+			#define DEBUG_ATTN(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
+				"INFO : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+			#define DEBUG_STACK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
+				"STACK: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#else
+			#define DEBUG_INFO(fmt, args...) fprintf(stderr,                                               \
+				"INFO : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+			#define DEBUG_ATTN(fmt, args...) fprintf(stderr, ZT_CYN                                        \
+				"ATTN : %17s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+			#define DEBUG_STACK(fmt, args...) fprintf(stderr, ZT_YEL                                       \
+				"STACK: %17s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+			#define DEBUG_BLANK(fmt, args...) fprintf(stderr,                                              \
+				"INFO : %17s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, ##args)
+	#endif
+	#else
+		#define DEBUG_INFO(fmt, args...)
+		#define DEBUG_BLANK(fmt, args...)
+		#define DEBUG_ATTN(fmt, args...)
+		#define DEBUG_STACK(fmt, args...)
+	#endif
+
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_TRANSFER
+		#if defined(__ANDROID__)
+			#define DEBUG_TRANS(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
+				"TRANS: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#else
+			#define DEBUG_TRANS(fmt, args...) fprintf(stderr, ZT_GRN "TRANS: %17s:%5d:%25s: " fmt \
+				"\n" ZT_RESET, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_TRANS(fmt, args...)
+	#endif
+
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_EXTRA
+		#if defined(__ANDROID__)
+			#define DEBUG_EXTRA(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
+				"EXTRA: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#else
+			#define DEBUG_EXTRA(fmt, args...) fprintf(stderr, \
+				"EXTRA: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+	#else
+		#define DEBUG_EXTRA(fmt, args...)
+	#endif
+
+	#if ZT_DEBUG_LEVEL >= ZT_MSG_FLOW
+		#if defined(__ANDROID__)
+			#define DEBUG_FLOW(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \ 
+				"FLOW : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+		#else
+			#define DEBUG_FLOW(fmt, args...) fprintf(stderr, "FLOW : %17s:%5d:%25s: " fmt "\n", \
+				ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+		#endif
+		#else
+			#define DEBUG_FLOW(fmt, args...)
+	#endif
+#endif // LIBZT_DEBUG
+#if !defined(LIBZT_DEBUG) // no output
+	#define DEBUG_TEST(fmt, args...)
+	#define DEBUG_ERROR(fmt, args...)
+	#define DEBUG_INFO(fmt, args...)
+	#define DEBUG_BLANK(fmt, args...)
+	#define DEBUG_ATTN(fmt, args...)
 	#define DEBUG_TRANS(fmt, args...)
-#endif
-
-#if ZT_DEBUG_LEVEL >= ZT_MSG_EXTRA
-	#if defined(__ANDROID__)
-		#define DEBUG_EXTRA(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-			"EXTRA: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-	#else
-		#define DEBUG_EXTRA(fmt, args...) fprintf(stderr, \
-			"EXTRA: %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-	#endif
-#else
 	#define DEBUG_EXTRA(fmt, args...)
-#endif
+	#define DEBUG_FLOW(fmt, args...)
-
-#if ZT_DEBUG_LEVEL >= ZT_MSG_FLOW
-	#if defined(__ANDROID__)
-		#define DEBUG_FLOW(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \ 
-			"FLOW : %17s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
-	#else
-		#define DEBUG_FLOW(fmt, args...) fprintf(stderr, "FLOW : %17s:%5d:%25s: " fmt "\n", \
-			ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
-	#endif
-	#else
-		#define DEBUG_FLOW(fmt, args...)
 #endif

make-liblwip.mk

@@ -84,9 +84,10 @@ CORE6FILES=$(LWIPDIR)/core/ipv6/ethip6.c \
 	$(LWIPDIR)/core/ipv6/dhcp6.c \
 	$(LWIPDIR)/core/ipv6/nd6.c
 	# APIFILES: The files which implement the sequential and socket APIs.
-APIFILES=$(LWIPDIR)/api/api_lib.c \
+APIFILES=$(LWIPDIR)/api/err.c
+#$(LWIPDIR)/api/api_lib.c \
 	$(LWIPDIR)/api/api_msg.c \
-	$(LWIPDIR)/api/err.c \
+	 \
 	$(LWIPDIR)/api/netbuf.c \
 	$(LWIPDIR)/api/netdb.c \
 	$(LWIPDIR)/api/netifapi.c \
@@ -99,8 +100,7 @@ SIXLOWPAN=$(LWIPDIR)/netif/lowpan6.c \
 # ARCHFILES: Architecture specific files.
 ARCHFILES=$(wildcard $(LWIPARCH)/*.c $(LWIPARCH)tapif.c $(LWIPARCH)/netif/list.c $(LWIPARCH)/netif/tcpdump.c)
 # LWIPFILES: All the above.
-LWIPFILES=$(COREFILES) $(NETIFFILES) $(ARCHFILES)
+LWIPFILES=$(COREFILES) $(NETIFFILES) $(ARCHFILES) $(APIFILES)
-#$(APIFILES)
 
 ifeq ($(LIBZT_IPV4),1)
 	LWIPFILES+=$(CORE4FILES)

make-linux.mk

@@ -73,6 +73,9 @@ CXXFLAGS=$(CFLAGS) -Wno-format -fno-rtti -std=c++11 -DZT_SOFTWARE_UPDATE_DEFAULT
 ifeq ($(LIBZT_SANITIZE),1)
 	SANFLAGS+=-x c++ -O -g -fsanitize=address -DASAN_OPTIONS=symbolize=1 -DASAN_SYMBOLIZER_PATH=$(shell which llvm-symbolizer)
 endif
+ifeq ($(LIBZT_DEBUG),1)
+	CXXFLAGS+=-DLIBZT_DEBUG
+endif
 
 INCLUDES+= -Iext \
 	-I$(ZTO)/osdep \
@@ -271,7 +274,7 @@ selftest:
 	@$(CXX) $(CXXFLAGS) $(SANFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__SELFTEST__ -o $(BUILD)/selftest $(UNIT_TEST_LIBS)
 	@./check.sh $(BUILD)/selftest
 nativetest:
-	@$(CXX) -v $(CXXFLAGS) $(SANFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__NATIVETEST__ -o $(BUILD)/nativetest
+	@$(CXX) $(CXXFLAGS) $(SANFLAGS) $(UNIT_TEST_INCLUDES) $(INCLUDES) test/selftest.cpp -D__NATIVETEST__ -o $(BUILD)/nativetest
 	@./check.sh $(BUILD)/nativetest
 
 	

make-mac.mk

@@ -74,6 +74,9 @@ CXXFLAGS=$(CFLAGS) -Wno-format -fno-rtti -std=c++11 -DZT_SOFTWARE_UPDATE_DEFAULT
 ifeq ($(LIBZT_SANITIZE),1)
 	SANFLAGS+=-x c++ -O -g -fsanitize=address -DASAN_OPTIONS=symbolize=1 -DASAN_SYMBOLIZER_PATH=$(shell which llvm-symbolizer)
 endif
+ifeq ($(LIBZT_DEBUG),1)
+	CXXFLAGS+=-DLIBZT_DEBUG
+endif
 
 INCLUDES+= -Iext \
 	-I$(ZTO)/osdep \

src/VirtualTap.cpp

@@ -529,7 +529,7 @@ namespace ZeroTier {
 	{
 		/* FIXME: There is a call to *_Connect for each send, we should probably figure out a better way to do this, 
 		possibly consult the stack for "connection" state */
-		
+
 		// TODO: flags
 		int err = 0;
 #if defined(STACK_PICO)
@@ -546,8 +546,12 @@ namespace ZeroTier {
 #endif
 #if defined(STACK_LWIP)
 		if(lwipstack) {
-			err = lwipstack->lwip_Connect(vs, addr, addrlen); // implicit
+			if((err = lwipstack->lwip_Connect(vs, addr, addrlen)) < 0) { // implicit
-			err = lwipstack->lwip_Write(vs, (void*)buf, len);
+				return err;
+			}
+			if((err = lwipstack->lwip_Write(vs, (void*)buf, len)) < 0) {
+				return err;
+			}
 		}
 #endif
 		return err;
@@ -586,14 +590,13 @@ namespace ZeroTier {
 		if(picostack) {
 			err = picostack->pico_Shutdown(vs, how);
 		}
-		return err;
 #endif
 #if defined(STACK_LWIP)
 		if(lwipstack) {
 			err = lwipstack->lwip_Shutdown(vs, how);
 		}
-		return err;
 #endif
+		return err;
 	}
 
 	void VirtualTap::Housekeeping()

src/picoTCP.cpp

@@ -264,16 +264,15 @@ namespace ZeroTier {
 			handle_general_failure();
 			return;
 		}
-
+		int r;		
-		int r, n;				
 		uint16_t port = 0;
 		union {
 			struct pico_ip4 ip4;
 			struct pico_ip6 ip6;
 		} peer;
-
+		
 		do {
-			n = 0;
+			int n = 0;
 			int avail = ZT_TCP_RX_BUF_SZ - vs->RXbuf->count();
 			if(avail) {
 				r = pico_socket_recvfrom(s, vs->RXbuf->get_buf(), ZT_STACK_SOCKET_RD_MAX, 
@@ -622,7 +621,7 @@ namespace ZeroTier {
 				Utils::ntoh(ethhdr->proto), beautify_eth_proto_nums(Utils::ntoh(ethhdr->proto)), flagbuf);
 		}
 		tap->_handler(tap->_arg,NULL,tap->_nwid,src_mac,dest_mac,
-			Utils::ntoh((uint16_t)ethhdr->proto),0, ((char*)buf) 
+			Utils::ntoh((uint16_t)ethhdr->proto), 0, ((char*)buf) 
 			+ sizeof(struct pico_eth_hdr),len - sizeof(struct pico_eth_hdr));
 		//_picostack_driver_lock.unlock();
 		return len;