Added stubs for poll/select, improved selftest/echotest, Added ringbuffer for more efficient RX/TX

include/Debug.hpp

@@ -86,14 +86,14 @@
 #endif
 
  #if ZT_DEBUG_LEVEL >= ZT_MSG_TEST
-  #define DEBUG_TEST(fmt, args...) fprintf(stderr, ZT_CYN "ZT_TEST [%d] : %16s:%5d:%25s: " fmt   \
+  #define DEBUG_TEST(fmt, args...) fprintf(stderr, ZT_CYN "TEST [%d] : %16s:%5d:%25s: " fmt   \
     "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
  #else
   #define DEBUG_ERROR(fmt, args...)
  #endif
 
  #if ZT_DEBUG_LEVEL >= ZT_MSG_ERROR
-  #define DEBUG_ERROR(fmt, args...) fprintf(stderr, ZT_RED "ZT_ERROR[%d] : %16s:%5d:%25s: " fmt   \
+  #define DEBUG_ERROR(fmt, args...) fprintf(stderr, ZT_RED "ERROR[%d] : %16s:%5d:%25s: " fmt   \
     "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
  #else
   #define DEBUG_ERROR(fmt, args...)
@@ -102,22 +102,22 @@
  #if ZT_DEBUG_LEVEL >= ZT_MSG_INFO
   #if defined(__ANDROID__)
     #define DEBUG_INFO(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
-      "ZT_INFO : %16s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "INFO : %16s:%5d:%20s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
     #define DEBUG_BLANK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
-      "ZT_INFO : %16s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "INFO : %16s:%5d:" fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
     #define DEBUG_ATTN(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,   \
-      "ZT_INFO : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "INFO : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
     #define DEBUG_STACK(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
-      "ZT_STACK: %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "STACK: %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
   #else
     #define DEBUG_INFO(fmt, args...) fprintf(stderr,                                                                   \
-      "ZT_INFO [%d] : %16s:%5d:%25s: " fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+      "INFO [%d] : %16s:%5d:%25s: " fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
     #define DEBUG_ATTN(fmt, args...) fprintf(stderr, ZT_CYN                                                            \
-      "ZT_ATTN [%d] : %16s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+      "ATTN [%d] : %16s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
     #define DEBUG_STACK(fmt, args...) fprintf(stderr, ZT_YEL                                                           \
-      "ZT_STACK[%d] : %16s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+      "STACK[%d] : %16s:%5d:%25s: " fmt "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
     #define DEBUG_BLANK(fmt, args...) fprintf(stderr,                                                                  \
-      "ZT_INFO [%d] : %16s:%5d:" fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, ##args)
+      "INFO [%d] : %16s:%5d:" fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, ##args)
   #endif
  #else
   #define DEBUG_INFO(fmt, args...)
@@ -129,9 +129,9 @@
  #if ZT_DEBUG_LEVEL >= ZT_MSG_TRANSFER
   #if defined(__ANDROID__)
     #define DEBUG_TRANS(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG,  \
-      "ZT_TRANS : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "TRANS : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
   #else
-    #define DEBUG_TRANS(fmt, args...) fprintf(stderr, ZT_GRN "ZT_TRANS[%ld] : %16s:%5d:%25s: " fmt \
+    #define DEBUG_TRANS(fmt, args...) fprintf(stderr, ZT_GRN "TRANS[%ld] : %16s:%5d:%25s: " fmt \
       "\n" ZT_RESET, ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
   #endif
  #else
@@ -141,10 +141,10 @@
  #if ZT_DEBUG_LEVEL >= ZT_MSG_EXTRA
    #if defined(__ANDROID__)
     #define DEBUG_EXTRA(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \
-      "ZT_EXTRA : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "EXTRA : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
   #else
     #define DEBUG_EXTRA(fmt, args...) fprintf(stderr, \
-      "ZT_EXTRA[%d] : %16s:%5d:%25s: " fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
+      "EXTRA[%d] : %16s:%5d:%25s: " fmt "\n", ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
   #endif
  #else
   #define DEBUG_EXTRA(fmt, args...)
@@ -153,9 +153,9 @@
 #if ZT_DEBUG_LEVEL >= ZT_MSG_FLOW
    #if defined(__ANDROID__)
     #define DEBUG_FLOW(fmt, args...) ((void)__android_log_print(ANDROID_LOG_VERBOSE, ZT_LOG_TAG, \ 
-      "ZT_FLOW : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
+      "FLOW : %16s:%5d:%25s: " fmt "\n", ZT_FILENAME, __LINE__, __FUNCTION__, ##args))
   #else
-    #define DEBUG_FLOW(fmt, args...) fprintf(stderr, "ZT_FLOW [%ld] : %16s:%5d:%25s: " fmt "\n", \
+    #define DEBUG_FLOW(fmt, args...) fprintf(stderr, "FLOW [%ld] : %16s:%5d:%25s: " fmt "\n", \
       ZT_THREAD_ID, ZT_FILENAME, __LINE__, __FUNCTION__, ##args)
   #endif
  #else

include/libzt.h

@@ -37,7 +37,7 @@
 #define ZT_PHY_POLL_INTERVAL               10  // ms
 #define ZT_ACCEPT_RECHECK_DELAY            100 // ms (for blocking zts_accept() calls)
 #define ZT_CONNECT_RECHECK_DELAY           100 // ms (for blocking zts_connect() calls)
-#define ZT_API_CHECK_INTERVAL              500 // ms
+#define ZT_API_CHECK_INTERVAL              100 // ms
 
 #define MAX_PICO_FRAME_RX_BUF_SZ           ZT_MAX_MTU * 128
 
@@ -46,7 +46,11 @@
 #define ZT_UDP_TX_BUF_SZ                   ZT_MAX_MTU
 #define ZT_UDP_RX_BUF_SZ                   ZT_MAX_MTU * 10
 
-#define ZT_SDK_RPC_DIR_PREFIX              "rpc.d"
+#define ZT_STACK_TCP_SOCKET_TX_SZ          2048
+#define ZT_STACK_TCP_SOCKET_RX_SZ          2048
+
+#define ZT_STACK_SOCKET_RD_MAX             2048
+#define ZT_STACK_SOCKET_WR_MAX             2048
 
 #define ZT_CORE_VERSION_MAJOR              1
 #define ZT_CORE_VERSION_MINOR              2
@@ -75,7 +79,7 @@
 // a short period of time by default as a precaution.
 
 #define ZT_SOCK_BEHAVIOR_LINGER            true
-#define ZT_SOCK_BEHAVIOR_LINGER_TIME       3  // s
+#define ZT_SOCK_BEHAVIOR_LINGER_TIME       10000 // ms
 
 // Wait time for socket closure if data is still present in the write queue
 #define ZT_SDK_CLTIME                      60
@@ -104,6 +108,7 @@
 #define ZT_SEND_SIG int fd, const void *buf, size_t len, int flags
 #define ZT_READ_SIG int fd, void *buf, size_t len
 #define ZT_WRITE_SIG int fd, const void *buf, size_t len
+#define ZT_SHUTDOWN_SIG int fd, int how
 #define ZT_SOCKET_SIG int socket_family, int socket_type, int protocol
 #define ZT_CONNECT_SIG int fd, const struct sockaddr *addr, socklen_t addrlen
 #define ZT_BIND_SIG int fd, const struct sockaddr *addr, socklen_t addrlen
@@ -111,6 +116,8 @@
 #define ZT_ACCEPT4_SIG int fd, struct sockaddr *addr, socklen_t *addrlen, int flags
 #define ZT_ACCEPT_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
 #define ZT_CLOSE_SIG int fd
+#define ZT_POLL_SIG struct pollfd *fds, nfds_t nfds, int timeout
+#define ZT_SELECT_SIG int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout
 #define ZT_GETSOCKNAME_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
 #define ZT_GETPEERNAME_SIG int fd, struct sockaddr *addr, socklen_t *addrlen
 #define ZT_FCNTL_SIG int fd, int cmd, int flags
@@ -326,6 +333,16 @@ int zts_getpeername(ZT_GETPEERNAME_SIG);
  */
 int zts_close(ZT_CLOSE_SIG);
 
+/**
+ * waits for one of a set of file descriptors to become ready to perform I/O.
+ */
+int zts_poll(ZT_POLL_SIG);
+
+/**
+ * monitor multiple file descriptors, waiting until one or more of the file descriptors become "ready"
+ */
+int zts_select(ZT_SELECT_SIG);
+
 /**
  * Issue file control commands on a socket
  */
@@ -367,6 +384,11 @@ int zts_read(ZT_READ_SIG);
  */
 int zts_write(ZT_WRITE_SIG);
 
+/*
+ * Sends a FIN segment 
+ */
+int zts_shutdown(ZT_SHUTDOWN_SIG);
+
 /****************************************************************************/
 /* SDK Socket API Helper functions/objects --- DONT CALL THESE DIRECTLY     */
 /****************************************************************************/
@@ -377,6 +399,11 @@ namespace ZeroTier
   extern ZeroTier::picoTCP *picostack;
 }
 
+/*
+ * Gets a pointer to a pico_socket given a file descriptor
+ */
+int zts_get_pico_socket(int fd, struct pico_socket *s);
+
 /**
  * Returns the number of sockets either already provisioned or waiting to be
  * Some network stacks may have a limit on the number of sockets that they can
@@ -397,6 +424,11 @@ int pico_ntimers();
  */
 void *zts_start_service(void *thread_id);
 
+/*
+ *
+ */
+void handle_general_failure();
+
 #include "Debug.hpp"
 
 #ifdef __cplusplus

make-linux.mk

@@ -67,7 +67,7 @@ ifeq ($(ZT_DEBUG),1)
 	# C25519 in particular is almost UNUSABLE in heavy testing without it.
 #ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS)
 else
-	CFLAGS?=-O2 -fstack-protector
+	CFLAGS?=-Ofast -g -fstack-protector
 	CFLAGS+=-Wall -fPIE -fvisibility=hidden -pthread $(INCLUDES) $(DEFS)
 	#CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIC -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
 	STRIP=strip
@@ -188,7 +188,7 @@ shared_jni_lib: picotcp $(ZTO_OBJS)
 
 UNIT_TEST_SRC_FILES := $(wildcard  $(UNIT_TEST_SRC_DIR)/*.cpp)
 UNIT_TEST_OBJ_FILES := $(addprefix $(TEST_BUILD_DIR)/,$(notdir $(UNIT_TEST_SRC_FILES:.cpp=)))
-UNIT_TEST_INCLUDES  := -Iinclude 
+UNIT_TEST_INCLUDES  := -Iinclude
 UNIT_TEST_LIBS      := -L$(BUILD) -lzt $(COMMON_LIBS)
 
 $(TEST_BUILD_DIR)/%: $(UNIT_TEST_SRC_DIR)/%.cpp

make-mac.mk

@@ -67,7 +67,7 @@ ifeq ($(ZT_DEBUG),1)
 	# C25519 in particular is almost UNUSABLE in heavy testing without it.
 #ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS)
 else
-	CFLAGS?=-O2 -fstack-protector
+	CFLAGS?=-Ofast -g -fstack-protector
 	CFLAGS+=-Wall -fPIE -fvisibility=hidden -pthread $(INCLUDES) $(DEFS)
 	#CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIC -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
 	STRIP=strip

src/Connection.hpp

@@ -39,6 +39,7 @@
 // SDK
 #include "libzt.h"
 #include "SocketTap.hpp"
+#include "RingBuffer.hpp"
 
 namespace ZeroTier {
 
@@ -47,9 +48,12 @@ namespace ZeroTier {
 	*/
 	struct Connection
 	{
+		int tot = 0;
+		RingBuffer<unsigned char> *TXbuf;
+		RingBuffer<unsigned char> *RXbuf;
+
 		Mutex _tx_m, _rx_m;
 
-		int pid;
 		PhySocket *sock;				
 		struct pico_socket *picosock;
 
@@ -57,12 +61,6 @@ namespace ZeroTier {
 		struct sockaddr_storage *local_addr; // Address we've bound to locally
 		struct sockaddr_storage *peer_addr;  // Address of connection call to remote host
 
-		// RX/TX buffers
-		int txsz = 0, rxsz = 0;
-		unsigned char txbuf[ZT_TCP_TX_BUF_SZ];
-		unsigned char rxbuf[ZT_TCP_RX_BUF_SZ];
-
-		int data_sock;
 		int socket_family, socket_type;
 
 		int app_fd; // provided to app for I/O
@@ -76,6 +74,10 @@ namespace ZeroTier {
 		std::time_t closure_ts;
 
 		Connection() {
+
+			TXbuf = new RingBuffer<unsigned char>(ZT_TCP_TX_BUF_SZ);
+			RXbuf = new RingBuffer<unsigned char>(ZT_TCP_RX_BUF_SZ);
+
 			closure_ts = -1;
 			ZT_PHY_SOCKFD_TYPE fdpair[2];
 			if(socketpair(PF_LOCAL, SOCK_STREAM, 0, fdpair) < 0) {
@@ -86,15 +88,18 @@ namespace ZeroTier {
 			}
 			sdk_fd = fdpair[0];
 			app_fd = fdpair[1];
+			//DEBUG_ERROR("sdk_fd = %d, app_fd = %d", sdk_fd, app_fd);
 
+/*
 			if(ZT_SOCK_BEHAVIOR_LINGER) {
 				struct linger so_linger;
 				so_linger.l_onoff = true;
 				so_linger.l_linger = ZT_SOCK_BEHAVIOR_LINGER_TIME;
-		        if(zts_setsockopt(app_fd, SOL_SOCKET, SO_LINGER, &so_linger, sizeof so_linger) < 0) {
+		        if(setsockopt(app_fd, SOL_SOCKET, SO_LINGER, &so_linger, sizeof so_linger) < 0) {
     				DEBUG_ERROR("error setsockopt (%d)", errno);
 		        }
 		    }
+*/
 		}
 		~Connection()
 		{

src/RingBuffer.hpp

@@ -0,0 +1,189 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_RINGBUFFER_HPP
+#define ZT_RINGBUFFER_HPP
+
+#include <memory.h>
+#include <algorithm>
+
+namespace ZeroTier {
+
+  template<typename T> class RingBuffer {
+
+  private:
+      T * buf;
+      size_t size;
+      size_t begin;
+      size_t end;
+      bool wrap;
+
+  public:
+      /**
+        * create a RingBuffer with space for up to size elements.
+        */
+      explicit RingBuffer(size_t size)
+              : size(size), 
+              begin(0),
+              end(0),
+              wrap(false)
+      {
+          buf = new T[size];
+      }
+
+      RingBuffer(const RingBuffer<T> & ring)
+      {
+          this(ring.size);
+          begin = ring.begin;
+          end = ring.end;
+          memcpy(buf, ring.buf, sizeof(T) * size);
+      }
+
+      ~RingBuffer()
+      {
+          delete[] buf;
+      }
+
+      // get a reference to the underlying buffer
+      T* get_buf()
+      {
+        return buf + begin;
+      }
+
+      // adjust buffer index pointer as if we copied data in
+      size_t produce(size_t n)
+      {
+          n = std::min(n, getFree());
+
+          if (n == 0) {
+              return n;
+          }
+
+          const size_t first_chunk = std::min(n, size - end);
+          end = (end + first_chunk) % size;
+
+          if (first_chunk < n) {
+              const size_t second_chunk = n - first_chunk;
+              end = (end + second_chunk) % size;
+          }
+
+          if (begin == end) {
+              wrap = true;
+          }
+
+          return n;
+      }
+
+      // adjust buffer index pointer as if we copied data out
+      size_t consume(size_t n)
+      {
+          n = std::min(n, count());
+
+          if (n == 0) {
+              return n;
+          }
+
+          if (wrap) {
+              wrap = false;
+          }
+
+          const size_t first_chunk = std::min(n, size - begin);
+          begin = (begin + first_chunk) % size;
+
+          if (first_chunk < n) {
+              const size_t second_chunk = n - first_chunk;
+              begin = (begin + second_chunk) % size;
+          }
+          return n;
+      }
+
+      size_t write(const T * data, size_t n)
+      {
+          n = std::min(n, getFree());
+
+          if (n == 0) {
+              return n;
+          }
+
+          const size_t first_chunk = std::min(n, size - end);
+          memcpy(buf + end, data, first_chunk * sizeof(T));
+          end = (end + first_chunk) % size;
+
+          if (first_chunk < n) {
+              const size_t second_chunk = n - first_chunk;
+              memcpy(buf + end, data + first_chunk, second_chunk * sizeof(T));
+              end = (end + second_chunk) % size;
+          }
+
+          if (begin == end) {
+              wrap = true;
+          }
+
+          return n;
+      }
+
+      size_t read(T * dest, size_t n)
+      {
+          n = std::min(n, count());
+
+          if (n == 0) {
+              return n;
+          }
+
+          if (wrap) {
+              wrap = false;
+          }
+
+          const size_t first_chunk = std::min(n, size - begin);
+          memcpy(dest, buf + begin, first_chunk * sizeof(T));
+          begin = (begin + first_chunk) % size;
+
+          if (first_chunk < n) {
+              const size_t second_chunk = n - first_chunk;
+              memcpy(dest + first_chunk, buf + begin, second_chunk * sizeof(T));
+              begin = (begin + second_chunk) % size;
+          }
+          return n;
+      }
+
+      size_t count() {
+          if (end == begin) {
+              return wrap ? size : 0;
+          } 
+          else if (end > begin) {
+              return end - begin;
+          } 
+          else {
+              return size + end - begin;
+          }
+      }
+
+      size_t getFree() {
+          return size - count();
+      }
+  };
+}
+#endif // ZT_RINGBUFFER_HPP

src/SocketTap.cpp

@@ -221,18 +221,21 @@ namespace ZeroTier {
 	
 	void SocketTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data, ssize_t len)
 	{
-		//DEBUG_INFO();
+		//DEBUG_ATTN("sock->fd=%d", _phy.getDescriptor(sock));
 		Connection *conn = (Connection*)*uptr;
 		if(!conn)
 			return;
-	    if(len)
+	    if(len){
+
 	        Write(conn, data, len);
+	    }
 	    return;
 	}
 
 	void SocketTap::phyOnUnixWritable(PhySocket *sock,void **uptr,bool stack_invoked)
 	{
 		DEBUG_INFO();
+		//exit(0);
 		if(sock)
 			Read(sock,uptr,stack_invoked);
 	}
@@ -270,11 +273,13 @@ namespace ZeroTier {
 	}
 
 	void SocketTap::Read(PhySocket *sock,void **uptr,bool stack_invoked) {
+		DEBUG_INFO();
 		if(picostack)
 			picostack->pico_Read(this, sock, (Connection*)uptr, stack_invoked);
 	}
 
 	void SocketTap::Write(Connection *conn, void *data, ssize_t len) {
+		//DEBUG_INFO();
 		if(picostack)
 			picostack->pico_Write(conn, data, len);
 	}
@@ -284,6 +289,7 @@ namespace ZeroTier {
 			DEBUG_ERROR("invalid connection");
 			return;
 		}
+		//DEBUG_INFO("A");
 		picostack->pico_Close(conn);
 		if(!conn->sock) {
 			// DEBUG_EXTRA("invalid PhySocket");
@@ -291,10 +297,22 @@ namespace ZeroTier {
 		}
 		// Here we assume _tcpconns_m is already locked by caller
 		// FIXME: is this assumption still valid
-		if(conn->sock)
+		if(conn->state==ZT_SOCK_STATE_LISTENING)
-			_phy.close(conn->sock, false);
+		{
-
+			//DEBUG_INFO("B");
+			// since we never wrapped this socket
+			DEBUG_INFO("in LISTENING state, no need to close in PhyIO");
+			return;
+		}
+		else
+		{
+			//DEBUG_INFO("C");
+			if(conn->sock)
+				_phy.close(conn->sock, false);
+		}
 		close(_phy.getDescriptor(conn->sock));
+
+		//DEBUG_INFO("D");
 		for(size_t i=0;i<_Connections.size();++i) {
 			if(_Connections[i] == conn){
 				// FIXME: double free issue exists here (potentially)

src/libzt.cpp

@@ -30,8 +30,6 @@ stack driver and core ZeroTier service to create a socket-like interface
 for applications to use. See also: include/libzt.h */
 
 #include <sys/socket.h>
-//#include <sys/ioctl.h>
-//#include <stropts.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
@@ -100,7 +98,7 @@ void zts_start(const char *path)
     ZeroTier::picostack = new ZeroTier::picoTCP();
     pico_stack_init();
 
-    DEBUG_INFO("path=%s", path);
+    //DEBUG_INFO("path=%s", path);
     if(path)
         ZeroTier::homeDir = path;
     pthread_t service_thread;
@@ -127,10 +125,14 @@ void zts_stop() {
 void zts_join(const char * nwid) {
     if(zt1Service) {
         std::string confFile = zt1Service->givenHomePath() + "/networks.d/" + nwid + ".conf";
-        if(!ZeroTier::OSUtils::mkdir(ZeroTier::netDir))
+        if(!ZeroTier::OSUtils::mkdir(ZeroTier::netDir)) {
             DEBUG_ERROR("unable to create: %s", ZeroTier::netDir.c_str());
-        if(!ZeroTier::OSUtils::writeFile(confFile.c_str(), ""))
+            handle_general_failure();
+        }
+        if(!ZeroTier::OSUtils::writeFile(confFile.c_str(), "")) {
             DEBUG_ERROR("unable to write network conf file: %s", confFile.c_str());
+            handle_general_failure();
+        }
         zt1Service->join(nwid);
     }
 }
@@ -140,10 +142,12 @@ void zts_join_soft(const char * filepath, const char * nwid) {
     std::string confFile = net_dir + std::string(nwid) + ".conf";
     if(!ZeroTier::OSUtils::mkdir(net_dir)) {
         DEBUG_ERROR("unable to create: %s", net_dir.c_str());
+        handle_general_failure();
     }
     if(!ZeroTier::OSUtils::fileExists(confFile.c_str(),false)) {
         if(!ZeroTier::OSUtils::writeFile(confFile.c_str(), "")) {
             DEBUG_ERROR("unable to write network conf file: %s", confFile.c_str());
+            handle_general_failure();
         }
     }
 }
@@ -344,7 +348,12 @@ Darwin:
 
 // int socket_family, int socket_type, int protocol
 int zts_socket(ZT_SOCKET_SIG) {
-    DEBUG_INFO();
+    errno = 0;
+    if(socket_family < 0 || socket_type < 0 || protocol < 0) {
+        errno = EINVAL;
+        return -1;
+    }
+    //DEBUG_INFO();
     int err = 0;
     if(!zt1Service) {
         DEBUG_ERROR("cannot create socket, no service running. call zts_start() first.");
@@ -377,22 +386,43 @@ int zts_socket(ZT_SOCKET_SIG) {
         if(socket_type == SOCK_DGRAM) {
             psock = pico_socket_open(
                 protocol_version, PICO_PROTO_UDP, &ZeroTier::picoTCP::pico_cb_socket_activity);
+            if(psock) { // configure size of UDP SND/RCV buffers
+                // TODO
+            }
         }
         if(socket_type == SOCK_STREAM) {
             psock = pico_socket_open(
                 protocol_version, PICO_PROTO_TCP, &ZeroTier::picoTCP::pico_cb_socket_activity);
+            if(psock) { // configure size of TCP SND/RCV buffers
+                int tx_buf_sz = ZT_STACK_TCP_SOCKET_TX_SZ;
+                int rx_buf_sz = ZT_STACK_TCP_SOCKET_RX_SZ;
+                int t_err = 0;
+
+                int value = 1;
+                pico_socket_setoption(psock, PICO_TCP_NODELAY, &value);
+
+
+                if((t_err = pico_socket_setoption(psock, PICO_SOCKET_OPT_SNDBUF, &tx_buf_sz)) < 0)
+                    DEBUG_ERROR("unable to set SNDBUF size, err = %d, pico_err = %d", t_err, pico_err);
+                if((t_err = pico_socket_setoption(psock, PICO_SOCKET_OPT_RCVBUF, &rx_buf_sz)) < 0)
+                    DEBUG_ERROR("unable to set RCVBUF size, err = %d, pico_err = %d", t_err, pico_err);
+               
+                if(ZT_SOCK_BEHAVIOR_LINGER) {
+                    int linger_time_ms = ZT_SOCK_BEHAVIOR_LINGER_TIME;
+                    if((t_err = pico_socket_setoption(psock, PICO_SOCKET_OPT_LINGER, &linger_time_ms)) < 0)
+                        DEBUG_ERROR("unable to set LINGER, err = %d, pico_err = %d", t_err, pico_err);
+                }
+            }
         }
-        // set up Unix Domain socketpair (used for data later on)
+        if(psock) {
-        if(psock) {     
             conn->socket_family = socket_family;
             conn->socket_type = socket_type;
             conn->picosock = psock;
-            memset(conn->rxbuf, 0, ZT_UDP_RX_BUF_SZ);
             ZeroTier::unmap[conn->app_fd] = conn;
             err = conn->app_fd; // return one end of the socketpair
         }
         else {
-            DEBUG_ERROR("failed to create pico_socket");
+            //DEBUG_ERROR("failed to create pico_socket");
             err = -1;
         }
     }
@@ -455,7 +485,7 @@ Linux:
 
 */
 int zts_connect(ZT_CONNECT_SIG) {
-    DEBUG_INFO("fd = %d", fd);
+    //DEBUG_INFO("fd = %d", fd);
     int err = 0;
     if(fd < 0) {
         errno = EBADF;
@@ -475,20 +505,22 @@ int zts_connect(ZT_CONNECT_SIG) {
         char ipstr[INET6_ADDRSTRLEN];
         memset(ipstr, 0, INET6_ADDRSTRLEN);
         ZeroTier::InetAddress iaddr;
+        int port = 0;
 
         if(conn->socket_family == AF_INET) {
             inet_ntop(AF_INET, 
                 (const void *)&((struct sockaddr_in *)addr)->sin_addr.s_addr, ipstr, INET_ADDRSTRLEN);
             iaddr.fromString(ipstr);
+            port = ((struct sockaddr_in*)addr)->sin_port;
         }
         if(conn->socket_family == AF_INET6) {
             inet_ntop(AF_INET6, 
                 (const void *)&((struct sockaddr_in6 *)addr)->sin6_addr.s6_addr, ipstr, INET6_ADDRSTRLEN);
             // TODO: This is a hack, determine a proper way to do this
             iaddr.fromString(ipstr + std::string("/88"));
+            port = ((struct sockaddr_in6*)addr)->sin6_port;
         }
-        //DEBUG_INFO("ipstr= %s", ipstr);
+        DEBUG_EXTRA("fd = %d, %s : %d", fd, ipstr, ntohs(port));
-        //DEBUG_INFO("iaddr= %s", iaddr.toString().c_str());
         tap = zt1Service->getTap(iaddr);
         if(!tap) {
             DEBUG_ERROR("no route to host");
@@ -498,8 +530,6 @@ int zts_connect(ZT_CONNECT_SIG) {
         else {
             // pointer to tap we use in callbacks from the stack
             conn->picosock->priv = new ZeroTier::ConnectionPair(tap, conn); 
-            //DEBUG_INFO("found appropriate SocketTap");
-            // Semantically: tap->stack->connect
             err = tap->Connect(conn, fd, addr, addrlen); 
             if(err == 0) {
                 tap->_Connections.push_back(conn); // Give this Connection to the tap we decided on
@@ -507,9 +537,8 @@ int zts_connect(ZT_CONNECT_SIG) {
             }
             // Wrap the socketpair we created earlier
             // For I/O loop participation and referencing the PhySocket's parent Connection in callbacks
-            conn->sock = tap->_phy.wrapSocket(conn->sdk_fd, conn);     
+            conn->sock = tap->_phy.wrapSocket(conn->sdk_fd, conn);  
-            //DEBUG_INFO("wrapping conn->sdk_fd = %d", conn->sdk_fd);   
+            //DEBUG_ERROR("sock->fd = %d", tap->_phy.getDescriptor(conn->sock));      
-            //DEBUG_INFO("         conn->app_fd = %d", conn->app_fd);     
         }
     }
     else {
@@ -552,6 +581,7 @@ int zts_connect(ZT_CONNECT_SIG) {
                 // FIXME: locking and unlocking so often might cause a performance bottleneck while outgoing connections
                 // are being established (also applies to accept())
                 usleep(ZT_CONNECT_RECHECK_DELAY * 1000);
+                //DEBUG_ERROR("waiting to connect...\n");
                 tap->_tcpconns_m.lock();
                 for(int i=0; i<tap->_Connections.size(); i++)
                 {
@@ -592,7 +622,6 @@ Darwin:
                             address space.
 */
 int zts_bind(ZT_BIND_SIG) {
-    DEBUG_EXTRA("fd = %d", fd);
     int err = 0;
     if(fd < 0) {
         errno = EBADF;
@@ -612,14 +641,19 @@ int zts_bind(ZT_BIND_SIG) {
         memset(ipstr, 0, INET6_ADDRSTRLEN);
         ZeroTier::InetAddress iaddr;
 
+        int port = 0;
+
         if(conn->socket_family == AF_INET) {
             inet_ntop(AF_INET, 
                 (const void *)&((struct sockaddr_in *)addr)->sin_addr.s_addr, ipstr, INET_ADDRSTRLEN);
+            port = ((struct sockaddr_in*)addr)->sin_port;
         }
         if(conn->socket_family == AF_INET6) {
             inet_ntop(AF_INET6, 
                 (const void *)&((struct sockaddr_in6 *)addr)->sin6_addr.s6_addr, ipstr, INET6_ADDRSTRLEN);
+            port = ((struct sockaddr_in6*)addr)->sin6_port;
         }
+        DEBUG_EXTRA("fd = %d, %s : %d", fd, ipstr, ntohs(port));
         iaddr.fromString(ipstr);
         tap = zt1Service->getTap(iaddr);
 
@@ -695,6 +729,7 @@ int zts_listen(ZT_LISTEN_SIG) {
     if(!err) {
         backlog = backlog > 128 ? 128 : backlog; // See: /proc/sys/net/core/somaxconn
         err = tap->Listen(conn, fd, backlog);
+        conn->state = ZT_SOCK_STATE_LISTENING;
         ZeroTier::_multiplexer_lock.unlock();
     }
     return err;
@@ -816,7 +851,7 @@ EPERM Firewall rules forbid connection.
 /*
     [--] [EBADF]            The argument s is not a valid descriptor.
     [  ] [ENOTSOCK]         The argument s is a file, not a socket.
-    [  ] [ENOPROTOOPT]      The option is unknown at the level indicated.
+    [--] [ENOPROTOOPT]      The option is unknown at the level indicated.
     [  ] [EFAULT]           The address pointed to by optval is not in a valid
                             part of the process address space.  For getsockopt(),
                             this error may also be returned if optlen is not in a
@@ -825,14 +860,28 @@ EPERM Firewall rules forbid connection.
 */
 int zts_setsockopt(ZT_SETSOCKOPT_SIG)
 {
-    //DEBUG_INFO("fd = %d", fd);
+    DEBUG_INFO("fd = %d", fd);
     int err = 0;
     if(fd < 0) {
         errno = EBADF;
         err = -1;
     }
+
+    // Disable Nagle's algorithm
+    struct pico_socket *p;
+    err = zts_get_pico_socket(fd, p);
+    if(p) {
+        int value = 1;
+        if((err = pico_socket_setoption(p, PICO_TCP_NODELAY, &value)) < 0) {
+            if(err == PICO_ERR_EINVAL) {
+                DEBUG_ERROR("error while disabling Nagle's algorithm");
+                errno = ENOPROTOOPT;
+                return -1;
+            }
+        }
+
+    }
     err = setsockopt(fd, level, optname, optval, optlen);
-    //DEBUG_INFO("err = %d", err);
     return err;
 }
 
@@ -976,27 +1025,36 @@ int zts_close(ZT_CLOSE_SIG)
                     }
 
                     if(blocking) {
-                        DEBUG_INFO("socket is blocking, waiting for write operations before closure");
+                        DEBUG_INFO("blocking, waiting for write operations before closure...");
                         for(int i=0; i<ZT_SDK_CLTIME; i++) {
-                            if(conn->txsz == 0)
+                            if(conn->TXbuf->count() == 0)
                                 break;
-                            sleep(1);
+                            usleep(ZT_API_CHECK_INTERVAL * 1000);
                         }
                     }
 
                     // For cases where data might still need to pass through the library
                     // before socket closure
+                    
+                    /*
                     if(ZT_SOCK_BEHAVIOR_LINGER) {
                         socklen_t optlen;
                         struct linger so_linger;
                         so_linger.l_linger = 0;                        
                         zts_getsockopt(fd, SOL_SOCKET, SO_LINGER, &so_linger, &optlen);
-                        if (so_linger.l_linger != 0) {
+                        //DEBUG_ERROR("fd = %d, value = %d", fd, so_linger.l_linger);
+                       // if (so_linger.l_linger != 0) {
                             DEBUG_EXTRA("lingering before closure for (%d) seconds...", so_linger.l_linger);
-                            sleep(so_linger.l_linger); // do the linger!
+                            sleep(3); // do the linger!
-                        }    
+                       // }    
-
                     }
+                    else
+                    {
+                        DEBUG_ERROR("LINGER NOT enabled");
+                    }
+                    */
+
+                    //DEBUG_INFO("s->state = %s", ZeroTier::picoTCP::beautify_pico_state(conn->picosock->state));
                     tap->Close(conn);
                     ZeroTier::fdmap.erase(fd);
                     err = 0;
@@ -1009,6 +1067,19 @@ int zts_close(ZT_CLOSE_SIG)
     return err;
 }
 
+//#define ZT_POLL_SIG struct pollfd *fds, nfds_t nfds, int timeout
+//#define ZT_SELECT_SIG int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout
+
+int zts_poll(ZT_POLL_SIG)
+{
+    return 0;
+}
+
+int zts_select(ZT_SELECT_SIG)
+{
+    return 0;
+}
+
 int zts_fcntl(ZT_FCNTL_SIG)
 {
     //DEBUG_INFO("fd = %d", fd);
@@ -1084,15 +1155,95 @@ ssize_t zts_recvmsg(ZT_RECVMSG_SIG)
 }
 
 int zts_read(ZT_READ_SIG) {
-    //DEBUG_EXTRA("fd = %d", fd);
+    //DEBUG_INFO("fd = %d", fd);
     return read(fd, buf, len);
 }
 
 int zts_write(ZT_WRITE_SIG) {
-    //DEBUG_EXTRA("fd = %d", fd);
+    //DEBUG_INFO("fd = %d", fd);
     return write(fd, buf, len);
 }
 
+int zts_shutdown(ZT_SHUTDOWN_SIG)
+{
+    DEBUG_INFO("fd = %d", fd);
+ 
+    int err = 0, mode = 0;
+    if(how == SHUT_RD) mode = PICO_SHUT_RD;
+    if(how == SHUT_WR) mode = PICO_SHUT_WR;
+    if(how == SHUT_RDWR) mode = PICO_SHUT_RDWR;
+
+    if(fd < 0) {
+        errno = EBADF;
+        err = -1;
+    }
+    else
+    {
+        if(!zt1Service) {
+            DEBUG_ERROR("cannot shutdown socket. service not started. call zts_start(path) first");
+            errno = EBADF;
+            err = -1;
+        }
+        else
+        {
+            ZeroTier::_multiplexer_lock.lock();
+            // First, look for for unassigned connections
+            ZeroTier::Connection *conn = ZeroTier::unmap[fd];
+            // Since we found an unassigned connection, we don't need to consult the stack or tap
+            // during closure - it isn't yet stitched into the clockwork
+            if(conn) // unassigned
+            {
+                DEBUG_ERROR("unassigned shutdown");
+                /*
+                   PICO_SHUT_RD
+                   PICO_SHUT_WR
+                   PICO_SHUT_RDWR
+                */
+                if((err = pico_socket_shutdown(conn->picosock, mode)) < 0)
+                    DEBUG_ERROR("error calling pico_socket_shutdown()");
+                delete conn;
+                ZeroTier::unmap.erase(fd);
+                // FIXME: Is deleting this correct behaviour?
+            }
+            else // assigned
+            {
+                std::pair<ZeroTier::Connection*, ZeroTier::SocketTap*> *p = ZeroTier::fdmap[fd];
+                if(!p) 
+                {
+                    DEBUG_ERROR("unable to locate connection pair.");
+                    errno = EBADF;
+                    err = -1;
+                }
+                else // found everything, begin closure
+                {
+                    conn = p->first;
+                    int f_err, blocking = 1;
+                    if ((f_err = fcntl(fd, F_GETFL, 0)) < 0) {
+                        DEBUG_ERROR("fcntl error, err = %s, errno = %d", f_err, errno);
+                        err = -1;
+                    } 
+                    else {
+                        blocking = !(f_err & O_NONBLOCK);
+                    }
+                    if(blocking) {
+                        DEBUG_INFO("blocking, waiting for write operations before shutdown...");
+                        for(int i=0; i<ZT_SDK_CLTIME; i++) {
+                            if(conn->TXbuf->count() == 0)
+                                break;
+                            usleep(ZT_API_CHECK_INTERVAL * 1000);
+                        }
+                    }
+
+                    if((err = pico_socket_shutdown(conn->picosock, mode)) < 0)
+                        DEBUG_ERROR("error calling pico_socket_shutdown()");
+                }
+            }
+            ZeroTier::_multiplexer_lock.unlock();
+        }
+    }
+    return err;
+}
+
 /****************************************************************************/
 /* SDK Socket API (Java Native Interface JNI)                               */
 /* JNI naming convention: Java_PACKAGENAME_CLASSNAME_METHODNAME             */
@@ -1355,6 +1506,46 @@ namespace ZeroTier {
 /* SDK Socket API Helper functions --- DON'T CALL THESE DIRECTLY            */
 /****************************************************************************/
 
+int zts_get_pico_socket(int fd, struct pico_socket *s)
+{
+    int err = 0;
+    if(!zt1Service) {
+        DEBUG_ERROR("cannot shutdown socket. service not started. call zts_start(path) first");
+        errno = EBADF;
+        err = -1;
+    }
+    else
+    {
+        ZeroTier::_multiplexer_lock.lock();
+        // First, look for for unassigned connections
+        ZeroTier::Connection *conn = ZeroTier::unmap[fd];
+        // Since we found an unassigned connection, we don't need to consult the stack or tap
+        // during closure - it isn't yet stitched into the clockwork
+        if(conn)
+        {
+            s = conn->picosock;
+            return 1; // unassigned
+        }
+        else // assigned
+        {
+            std::pair<ZeroTier::Connection*, ZeroTier::SocketTap*> *p = ZeroTier::fdmap[fd];
+            if(!p) 
+            {
+                DEBUG_ERROR("unable to locate connection pair.");
+                errno = EBADF;
+                err = -1;
+            }
+            else // found everything, begin closure
+            {
+                s = p->first->picosock;
+                return 0;
+            }
+        }
+        ZeroTier::_multiplexer_lock.unlock();
+    }
+    return err;
+}
+
 int zts_nsockets()
 {
     ZeroTier::_multiplexer_lock.unlock();
@@ -1391,6 +1582,7 @@ void *zts_start_service(void *thread_id) {
             if ((*pi != ".")&&(*pi != "..")) {
                 if (!ZeroTier::OSUtils::mkdir(ptmp)) {
                     DEBUG_ERROR("home path does not exist, and could not create");
+                    handle_general_failure();
                     perror("error\n");
                 }
             }
@@ -1398,13 +1590,9 @@ void *zts_start_service(void *thread_id) {
     }
     else {
         DEBUG_ERROR("homeDir is empty, could not construct path");
+        handle_general_failure();
         return NULL;
     }
-    // rpc dir
-    // if(!ZeroTier::OSUtils::mkdir(ZeroTier::homeDir + "/" + ZT_SDK_RPC_DIR_PREFIX)) {
-    //   DEBUG_ERROR("unable to create dir: " ZT_SDK_RPC_DIR_PREFIX);
-    //    return NULL;
-    //}
 
     // Generate random port for new service instance
     unsigned int randp = 0;
@@ -1445,6 +1633,13 @@ void *zts_start_service(void *thread_id) {
     return NULL;
 }
 
+void handle_general_failure() {
+#ifdef ZT_EXIT_ON_GENERAL_FAIL
+    DEBUG_ERROR("exiting (ZT_EXIT_ON_GENERAL_FAIL==1)");
+    //exit(-1);
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

src/picoTCP.cpp

@@ -42,7 +42,7 @@
 #include "Utilities.hpp"
 #include "SocketTap.hpp"
 #include "picoTCP.hpp"
-//#include "RingBuffer.hpp"
+#include "RingBuffer.hpp"
 
 // ZT
 #include "Utils.hpp"
@@ -51,6 +51,7 @@
 #include "Constants.hpp"
 #include "Phy.hpp"
 
+
 extern "C" int pico_stack_init(void);
 extern "C" void pico_stack_tick(void);
 
@@ -83,7 +84,6 @@ struct pico_socket * pico_socket_accept(PICO_SOCKET_ACCEPT_SIG);
 
 namespace ZeroTier {
 
-	// TODO: Determine why stack interrupt code fails when picodev is a mmember of a SocketTap
 	struct pico_device picodev;
 
 	bool picoTCP::pico_init_interface(SocketTap *tap, const InetAddress &ip)
@@ -102,6 +102,7 @@ namespace ZeroTier {
 			    tap->_mac.copyTo(mac, PICO_SIZE_ETH);
 			    if(pico_device_init(&picodev, "pz", mac) != 0) {
 			        DEBUG_ERROR("dev init failed");
+			        handle_general_failure();
 			        return false;
 			    }
 				tap->picodev_initialized = true;
@@ -141,38 +142,69 @@ namespace ZeroTier {
 		}
 	}
 
+	// from stack socket to app socket
 	void picoTCP::pico_cb_tcp_read(ZeroTier::SocketTap *tap, struct pico_socket *s)
 	{
-		//DEBUG_INFO();
 		Connection *conn = (Connection*)((ConnectionPair*)(s->priv))->conn;
-		if(conn) {
+		Mutex::Lock _l(conn->_rx_m);
-			int r;				
+		
-			uint16_t port = 0;
+		if(!conn || !tap) {
-			union {
+    		DEBUG_ERROR("invalid tap or conn");
-		        struct pico_ip4 ip4;
+    		handle_general_failure();
-		        struct pico_ip6 ip6;
+    		return;
-		    } peer;
+    	}
-			do {
+
-				int avail = ZT_TCP_RX_BUF_SZ - conn->rxsz;
+		int r, n;				
-				if(avail) {
+		uint16_t port = 0;
-		            r = pico_socket_recvfrom(s, conn->rxbuf + (conn->rxsz), ZT_SDK_MTU, 
+		union {
-		            	(void *)&peer.ip4.addr, &port);
+	        struct pico_ip4 ip4;
-		            if (r > 0)
+	        struct pico_ip6 ip6;
-		                conn->rxsz += r;
+	    } peer;
-		            picostack->pico_Read(tap, conn->sock, conn, true);
+
-		            //DEBUG_INFO("r = %d, conn->rxsz=%d, conn=%p, conn->sock = %p", r, conn->rxsz, conn, conn->sock);
+		do {
-	        	}
+			n = 0;
-	        	else
+			//DEBUG_INFO("RXbuf->count() = %d", conn->RXbuf->count());
-	        		DEBUG_ERROR("not enough space left on I/O RX buffer for pico_socket(%p)", s);
+			int avail = ZT_TCP_RX_BUF_SZ - conn->RXbuf->count();
-            }
+			if(avail) {
-        	while(r > 0);
+
-        	return;
+	            r = pico_socket_recvfrom(s, conn->RXbuf->get_buf(), ZT_STACK_SOCKET_RD_MAX, 
+	            	(void *)&peer.ip4.addr, &port);
+	            
+	            conn->tot += r;
+
+	            if (r > 0)
+	            {
+	            	conn->RXbuf->produce(r);
+	            	//DEBUG_INFO("RXbuf->count() = %d", conn->RXbuf->count());
+	            	n = tap->_phy.streamSend(conn->sock, conn->RXbuf->get_buf(), r);
+
+	            	if(n>0)
+	            		conn->RXbuf->consume(n);
+	            	//DEBUG_INFO("pico_recv = %d, streamSend = %d, rxsz = %d, tot = %d", r, n, conn->RXbuf->count(), conn->tot);
+
+	            	//DEBUG_TRANS("[ TCP RX <- STACK] :: conn = %p, len = %d", conn, n);
+	            }
+
+	        	if(conn->RXbuf->count() == 0) {
+					tap->_phy.setNotifyWritable(conn->sock, false);
+				}
+				else {
+	        		tap->_phy.setNotifyWritable(conn->sock, true);	
+				}				
+        	}
+        	else {
+        		//tap->_phy.setNotifyWritable(conn->sock, false);
+        		DEBUG_ERROR("not enough space left on I/O RX buffer for pico_socket(%p)", s);
+        		handle_general_failure();
+        	}
         }
-		DEBUG_ERROR("invalid connection");
+    	while(r > 0);
 	}
 
+	// from stack socket to app socket
 	void picoTCP::pico_cb_udp_read(SocketTap *tap, struct pico_socket *s)
 	{
+		/*
 		DEBUG_INFO();
 		Connection *conn = (Connection*)((ConnectionPair*)(s->priv))->conn;
 		Mutex::Lock _l(conn->_rx_m);
@@ -216,6 +248,7 @@ namespace ZeroTier {
 	        }
 	        if (r < 0) {
             	DEBUG_ERROR("unable to read from picosock=%p", s);
+            	handle_general_failure();
     		} 
 	        tap->_rx_buf_m.unlock(); 
 
@@ -224,39 +257,43 @@ namespace ZeroTier {
         	//DEBUG_EXTRA(" Copied onto rxbuf (%d) from stack socket", r);
         	return;
 		}
+		*/
 	}
 
 	void picoTCP::pico_cb_tcp_write(SocketTap *tap, struct pico_socket *s)
 	{
-		//DEBUG_INFO();
 		Connection *conn = (Connection*)((ConnectionPair*)(s->priv))->conn;
-		Mutex::Lock _l(conn->_tx_m);
+	    Mutex::Lock _l(conn->_tx_m);
-		if(!conn) {
+	   	if(!conn) {
 			DEBUG_ERROR("invalid connection");
+			handle_general_failure();
 			return;
 		}
-		//DEBUG_INFO("conn      = %p", conn);
+	   	int txsz = conn->TXbuf->count();
-		//DEBUG_INFO("conn.txsz = %d", conn->txsz);
+	   	if(txsz <= 0)
-		if(!conn->txsz)
+	   		return;
-			return;
+	    //DEBUG_INFO("TXbuf->count() = %d", conn->TXbuf->count());
-		// Only called from a locked context, no need to lock anything
-		if(conn->txsz > 0) {
-			int r, max_write_len = conn->txsz < ZT_SDK_MTU ? conn->txsz : ZT_SDK_MTU;
-			if((r = pico_socket_write(s, &conn->txbuf, max_write_len)) < 0) {
-				DEBUG_ERROR("unable to write to picosock=%p", s);
-				return;
-			}
-			int sz = (conn->txsz)-r;
-            if(sz)
-                memmove(&conn->txbuf, (conn->txbuf+r), sz);
-            conn->txsz -= r;
-            //DEBUG_INFO("conn->txsz = %d, r = %d, sz = %d", conn->txsz, r, sz);
 
-            #if DEBUG_LEVEL >= MSG_TRANSFER
+		int r, max_write_len = std::min(std::min(txsz, ZT_SDK_MTU),ZT_STACK_SOCKET_WR_MAX);
-            	int max = conn->socket_type == SOCK_STREAM ? ZT_TCP_TX_BUF_SZ : ZT_UDP_TX_BUF_SZ;
+	    if((r = pico_socket_write(conn->picosock, conn->TXbuf->get_buf(), max_write_len)) < 0) {
-            	DEBUG_TRANS("[ TCP TX -> STACK] :: conn = %p, len = %d", conn, r);
+	    	DEBUG_ERROR("unable to write to picosock=%p, r=%d", conn->picosock, r);
-        	#endif
+	    	handle_general_failure();
-		}
+	    	return;
+	    }
+	   	if(conn->socket_type == SOCK_STREAM) {
+	    	//DEBUG_TRANS("[ TCP TX -> STACK] :: conn = %p, len = %d", conn, r);
+	    }
+	   	if(conn->socket_type == SOCK_DGRAM) {
+	    	//DEBUG_TRANS("[ UDP TX -> STACK] :: conn = %p, len = %d", conn, r);
+	    }
+	    if(r == 0) {
+	    	// This is a peciliarity of the picoTCP network stack, if we receive no error code, and the size of 
+	    	// the byte stream written is 0, this is an indication that the buffer for this pico_socket is too small
+	    	// DEBUG_ERROR("pico_socket buffer is too small (adjust ZT_STACK_SOCKET_TX_SZ, ZT_STACK_SOCKET_RX_SZ)");
+	    	// handle_general_failure(); 
+	    }
+	    if(r>0)
+	    	conn->TXbuf->consume(r);
 	}
 
 	void picoTCP::pico_cb_socket_activity(uint16_t ev, struct pico_socket *s)
@@ -267,11 +304,13 @@ namespace ZeroTier {
     	Connection *conn = (Connection*)((ConnectionPair*)(s->priv))->conn;
     	if(!tap || !conn) {
     		DEBUG_ERROR("invalid tap or conn");
+    		handle_general_failure();
     		return;
     	}
     	int err = 0;
         if(!conn) {
         	DEBUG_ERROR("invalid connection");
+        	handle_general_failure();
         	return;
         }
         // PICO_SOCK_EV_CONN - triggered when connection is established (TCP only). This event is
@@ -286,10 +325,7 @@ namespace ZeroTier {
 				uint16_t port;
 	            struct pico_socket *client_psock = pico_socket_accept(s, &peer, &port);
 	            if(!client_psock) {
-	            	if(pico_err == PICO_ERR_EINVAL)
+	            	DEBUG_ERROR("pico_err=%s, picosock=%p", beautify_pico_error(pico_err), s);
-	            		DEBUG_ERROR("pico_err = PICO_ERR_EINVAL");
-	            	if(pico_err == PICO_ERR_EAGAIN)
-	            		DEBUG_ERROR("pico_err = PICO_ERR_EAGAIN");
 	            	return;
 				}			
 
@@ -305,8 +341,27 @@ namespace ZeroTier {
 				newConn->picosock->priv = new ConnectionPair(tap,newConn);
 				tap->_Connections.push_back(newConn);
 				conn->_AcceptedConnections.push(newConn);
+
+
+                int value = 1;
+                pico_socket_setoption(newConn->picosock, PICO_TCP_NODELAY, &value);
+                
+				if(ZT_SOCK_BEHAVIOR_LINGER) {
+                    int linger_time_ms = ZT_SOCK_BEHAVIOR_LINGER_TIME;
+                    int t_err = 0;
+                	if((t_err = pico_socket_setoption(newConn->picosock, PICO_SOCKET_OPT_LINGER, &linger_time_ms)) < 0)
+                   		DEBUG_ERROR("unable to set LINGER size, err = %d, pico_err = %d, app_fd=%d, sdk_fd=%d", t_err, pico_err, conn->app_fd, conn->sdk_fd);
+           		}
+/*
+               	linger_time_ms = 0;
+                if((t_err = pico_socket_getoption(newConn->picosock, PICO_SOCKET_OPT_LINGER, &linger_time_ms)) < 0)
+                   DEBUG_ERROR("unable to set LINGER size, err = %d, pico_err = %d", t_err, pico_err);
+				DEBUG_TEST("getting linger = %d", linger_time_ms);
+*/
 				// For I/O loop participation and referencing the PhySocket's parent Connection in callbacks
 				newConn->sock = tap->_phy.wrapSocket(newConn->sdk_fd, newConn);
+				//DEBUG_ERROR("sock->fd = %d", tap->_phy.getDescriptor(newConn->sock));      
+
 			}
 			if(conn->state != ZT_SOCK_STATE_LISTENING) {
 				// set state so socket multiplexer logic will pick this up
@@ -317,7 +372,7 @@ namespace ZeroTier {
         // PICO_SOCK_EV_FIN - triggered when the socket is closed. No further communication is
 		// possible from this point on the socket.
         if (ev & PICO_SOCK_EV_FIN) {
-            // DEBUG_EXTRA("PICO_SOCK_EV_FIN (socket closed), picosock=%p, conn=%p", s, conn);
+            //DEBUG_EXTRA("PICO_SOCK_EV_FIN (socket closed), picosock=%p, conn=%p, app_fd=%d, sdk_fd=%d", s, conn, conn->app_fd, conn->sdk_fd);
             conn->closure_ts = std::time(nullptr);	
         }
 
@@ -327,7 +382,7 @@ namespace ZeroTier {
         		DEBUG_ERROR("PICO_ERR_ECONNRESET");
         		conn->state = PICO_ERR_ECONNRESET;
         	}
-            // DEBUG_INFO("PICO_SOCK_EV_ERR (socket error received) err=%d, picosock=%p", pico_err, s);
+            DEBUG_ERROR("PICO_SOCK_EV_ERR, err=%s, picosock=%p, app_fd=%d, sdk_fd=%d", beautify_pico_error(pico_err), s, conn->app_fd, conn->sdk_fd); 
         }
         // PICO_SOCK_EV_CLOSE - triggered when a FIN segment is received (TCP only). This event
 		// indicates that the oher endpont has closed the connection, so the local TCP layer is only
@@ -336,7 +391,7 @@ namespace ZeroTier {
 		// allowing new data to be sent in the TCP CLOSE WAIT state.
         if (ev & PICO_SOCK_EV_CLOSE) {
             err = pico_socket_close(s);
-            // DEBUG_INFO("PICO_SOCK_EV_CLOSE (socket closure) err = %d, picosock=%p, conn=%p", err, s, conn);
+            //DEBUG_INFO("PICO_SOCK_EV_CLOSE (socket closure) err = %d, picosock=%p, conn=%p, app_fd=%d, sdk_fd=%d", err, s, conn, conn->app_fd, conn->sdk_fd);            
             conn->closure_ts = std::time(nullptr);	
             return;
         }
@@ -361,6 +416,7 @@ namespace ZeroTier {
     	SocketTap *tap = (SocketTap*)(dev->tap);
     	if(!tap) {
     		DEBUG_ERROR("invalid dev->tap");
+    		handle_general_failure();
     		return ZT_ERR_GENERAL_FAILURE;
     	}
         struct pico_eth_hdr *ethhdr;
@@ -374,12 +430,14 @@ namespace ZeroTier {
         return len;
     }
 
+    // receive frames from zerotier virtual wire and copy them to a guarded buffer awaiting placement into network stack
     void picoTCP::pico_rx(SocketTap *tap, const MAC &from,const MAC &to,unsigned int etherType,
     	const void *data,unsigned int len)
 	{
-		//DEBUG_INFO("len = %d", len);
+		DEBUG_INFO("len = %d", len);
 		if(!tap) {
     		DEBUG_ERROR("invalid tap");
+    		handle_general_failure();
     		return;
     	}
 		// Since picoTCP only allows the reception of frames from within the polling function, we
@@ -408,11 +466,13 @@ namespace ZeroTier {
 		//DEBUG_FLOW("[ ZWIRE -> FBUF ] Move FRAME(sz=%d) into FBUF(sz=%d), data_len=%d", newlen, tap->pico_frame_rxbuf_tot, len);
 	}
 
+	// feed frames on the guarded RX buffer (from zerotier virtual wire) into the network stack
     int pico_eth_poll(struct pico_device *dev, int loop_score)
     {
     	SocketTap *tap = (SocketTap*)(dev->tap);
     	if(!tap) {
     		DEBUG_ERROR("invalid dev->tap");
+    		handle_general_failure();
     		return ZT_ERR_GENERAL_FAILURE;
     	}
         // FIXME: The copy logic and/or buffer structure should be reworked for better performance after the BETA
@@ -434,8 +494,10 @@ namespace ZeroTier {
             	//DEBUG_INFO("recv = %d", err);
                 tap->pico_frame_rxbuf_tot-=len;
             }
-            else
+            else {
             	DEBUG_ERROR("Invalid frame size (%d). Exiting.",len);
+            	handle_general_failure();
+            }
             loop_score--;
         }
         return loop_score;
@@ -445,6 +507,7 @@ namespace ZeroTier {
     {		
     	if(!conn || !conn->picosock) {
     		DEBUG_ERROR("invalid conn or conn->picosock");
+    		handle_general_failure();
     		return ZT_ERR_GENERAL_FAILURE;
     	}
 		int err = 0;
@@ -483,6 +546,7 @@ namespace ZeroTier {
     	//DEBUG_INFO();
     	if(!conn || !conn->picosock) {
     		DEBUG_ERROR("invalid conn or conn->picosock");
+    		handle_general_failure();
     		return ZT_ERR_GENERAL_FAILURE;
     	}
     	int err = 0;
@@ -492,9 +556,8 @@ namespace ZeroTier {
 			struct sockaddr_in *in4 = (struct sockaddr_in*)addr;
 			char ipv4_str[INET_ADDRSTRLEN];
 			inet_ntop(AF_INET, (const void *)&in4->sin_addr.s_addr, ipv4_str, INET_ADDRSTRLEN);
-			uint32_t ipval = 0;
 			pico_string_to_ipv4(ipv4_str, &(zaddr.addr));
-			DEBUG_EXTRA("addr=%s:%d", ipv4_str, Utils::ntoh(in4->sin_port));
+			//DEBUG_EXTRA("addr=%s:%d", ipv4_str, Utils::ntoh(in4->sin_port));
 			err = pico_socket_bind(conn->picosock, &zaddr, (uint16_t *)&(in4->sin_port));
 		}
 		if(conn->socket_family == AF_INET6) { 
@@ -504,7 +567,7 @@ namespace ZeroTier {
 			inet_ntop(AF_INET6, &(in6->sin6_addr), ipv6_str, INET6_ADDRSTRLEN);
 			// TODO: This isn't proper
 	    	pico_string_to_ipv6("::", pip6.addr);
-       		DEBUG_EXTRA("addr=%s:%d", ipv6_str, Utils::ntoh(in6->sin6_port));
+       		//DEBUG_EXTRA("addr=%s:%d", ipv6_str, Utils::ntoh(in6->sin6_port));
 			err = pico_socket_bind(conn->picosock, &pip6, (uint16_t *)&(in6->sin6_port));
 		}
 		if(err < 0) {
@@ -535,6 +598,7 @@ namespace ZeroTier {
     	//DEBUG_INFO();
     	if(!conn || !conn->picosock) {
     		DEBUG_ERROR("invalid conn or conn->picosock");
+    		handle_general_failure();
     		return ZT_ERR_GENERAL_FAILURE;
     	}
     	int err = 0;
@@ -559,6 +623,7 @@ namespace ZeroTier {
     {
     	if(!conn) {
     		DEBUG_ERROR("invalid conn");
+    		handle_general_failure();
     		return NULL;
     	}
     	// Retreive first of queued Connections from parent connection
@@ -572,8 +637,12 @@ namespace ZeroTier {
 
     void picoTCP::pico_Read(SocketTap *tap, PhySocket *sock, Connection* conn, bool stack_invoked)
     {
+    	DEBUG_INFO();
+    	//exit(0);
+    	/*
     	if(!conn || !tap || !conn) {
     		DEBUG_ERROR("invalid tap, sock, or conn");
+    		handle_general_failure();
     		return;
     	}
     	//DEBUG_INFO();
@@ -625,10 +694,9 @@ namespace ZeroTier {
 			// Notify ZT I/O loop that it has new buffer contents
 			if(n) {
 				if(conn->socket_type==SOCK_STREAM) {
-					#if DEBUG_LEVEL >= MSG_TRANSFER
+					//#if DEBUG_LEVEL >= MSG_TRANSFER
-						float max = conn->socket_type == SOCK_STREAM ? (float)ZT_TCP_RX_BUF_SZ : (float)ZT_UDP_RX_BUF_SZ;
+		            //	DEBUG_TRANS("[ TCP RX <- STACK] :: conn = %p, len = %d", conn, n);
-		            	DEBUG_TRANS("[ TCP RX <- STACK] :: conn = %p, len = %d", conn, n);
+					//#endif
-					#endif
 	        	}
 	        	if(conn->rxsz == 0) {
 					tap->_phy.setNotifyWritable(sock, false);
@@ -646,13 +714,17 @@ namespace ZeroTier {
             tap->_rx_buf_m.unlock();
         }
         // DEBUG_FLOW("[ ZTSOCK <- RXBUF] Emitted (%d) from RXBUF(%d) to socket", tot, conn->rxsz);
+        */
     }
 
     void picoTCP::pico_Write(Connection *conn, void *data, ssize_t len)
     {
-    	Mutex::Lock _l(conn->_tx_m);
+	    // TODO: Add RingBuffer overflow checks
-    	if(len <= 0) {
+	    //DEBUG_INFO("conn=%p, len = %d", conn, len);
-    		DEBUG_ERROR("invalid write length");
+	    Mutex::Lock _l(conn->_tx_m);
+		if(len <= 0) {
+    		DEBUG_ERROR("invalid write length (len=%d)", len);
+    		handle_general_failure();
     		return;
     	}
     	if(conn->picosock->state & PICO_SOCKET_STATE_CLOSED){
@@ -660,45 +732,41 @@ namespace ZeroTier {
     		return;
     	}
     	if(!conn) {
-    		DEBUG_ERROR("invalid connection");
+    		DEBUG_ERROR("invalid connection (len=%d)", len);
+    		handle_general_failure();
     		return;
     	}
-    	if(conn->txsz + len >= ZT_TCP_TX_BUF_SZ) {
+
+    	int original_txsz = conn->TXbuf->count();
+
+    	if(original_txsz + len >= ZT_TCP_TX_BUF_SZ) {
+    		DEBUG_ERROR("txsz = %d, len = %d", original_txsz, len);
     		DEBUG_ERROR("TX buffer is too small, try increasing ZT_TCP_TX_BUF_SZ in libzt.h");
-    		return;
+    		exit(0);
     	}
 
-    	// DEBUG_INFO("conn->txsz = %d, len = %d", conn->txsz, len);
+	    int buf_w = conn->TXbuf->write((const unsigned char*)data, len);
-    	unsigned char *buf = (unsigned char*)data;
+	    //DEBUG_INFO("TXbuf->count() = %d", conn->TXbuf->count());
-    	memcpy(conn->txbuf + conn->txsz, buf, len);
+	   	int txsz = conn->TXbuf->count();
-        conn->txsz += len;
 
-		//DEBUG_INFO("conn = %p, conn->picosock = %p", conn, conn->picosock);		
+	   	//if(original_txsz > 0)
-		if(!conn || !conn->picosock) {
+	   	//	return; // don't write here, we already have stuff in the queue, a callback will handle it
-			DEBUG_ERROR("invalid conn or conn->picosock");
+	   	
-			return;
+		int r, max_write_len = std::min(std::min(txsz, ZT_SDK_MTU),ZT_STACK_SOCKET_WR_MAX);
-		}
+		//int buf_r = conn->TXbuf->read(conn->tmptxbuf, max_write_len);
-		int max, r, max_write_len = conn->txsz < ZT_SDK_MTU ? conn->txsz : ZT_SDK_MTU;
+	    
-	    if((r = pico_socket_write(conn->picosock, &conn->txbuf, max_write_len)) < 0) {
+	    if((r = pico_socket_write(conn->picosock, conn->TXbuf->get_buf(), max_write_len)) < 0) {
 	    	DEBUG_ERROR("unable to write to picosock=%p, r=%d", conn->picosock, r);
 	    	return;
 	    }
-	    // adjust buffer
-	    int sz = (conn->txsz)-r;
-	   	if(sz)
-	   	{
-	   		memmove(&conn->txbuf, (conn->txbuf+r), sz);
-	   	}
-		conn->txsz -= r;
-	   	
 	   	if(conn->socket_type == SOCK_STREAM) {
-	   		max = ZT_TCP_TX_BUF_SZ;
+	    	//DEBUG_TRANS("[ TCP TX -> STACK] :: conn = %p, len = %d", conn, r);
-	    	DEBUG_TRANS("[ TCP TX -> STACK] :: conn = %p, len = %d", conn, r);
 	    }
 	   	if(conn->socket_type == SOCK_DGRAM) {
-	   		max = ZT_UDP_TX_BUF_SZ;
+	    	//DEBUG_TRANS("[ UDP TX -> STACK] :: conn = %p, len = %d", conn, r);
-	    	DEBUG_TRANS("[ UDP TX -> STACK] :: conn = %p, len = %d", conn, r);
 	    }
+	    if(r>0)
+	    	conn->TXbuf->consume(r);
     }
 
     int picoTCP::pico_Close(Connection *conn)
@@ -717,54 +785,169 @@ namespace ZeroTier {
     	return err;
     }
 
-	int beautify_pico_error(int err)
+	char *picoTCP::beautify_pico_error(int err)
 	{
-		return 0;
+		if(err==  0) return (char*)"PICO_ERR_NOERR";
-		/*
+		if(err==  1) return (char*)"PICO_ERR_EPERM";
-		switch(err){
+		if(err==  2) return (char*)"PICO_ERR_ENOENT";
-			PICO_ERR_NOERR = 0,
+		// ...
-	    PICO_ERR_EPERM = 1,
+		if(err==  4) return (char*)"PICO_ERR_EINTR";
-	    PICO_ERR_ENOENT = 2,
+		if(err==  5) return (char*)"PICO_ERR_EIO";
-	    
+		if(err==  6) return (char*)"PICO_ERR_ENXIO";
-	    PICO_ERR_EINTR = 4,
+		// ...
-	    PICO_ERR_EIO = 5,
+		if(err== 11) return (char*)"PICO_ERR_EAGAIN";
-	    PICO_ERR_ENXIO = 6,
+		if(err== 12) return (char*)"PICO_ERR_ENOMEM";
-	    
+		if(err== 13) return (char*)"PICO_ERR_EACCESS";
-	    PICO_ERR_EAGAIN = 11,
+		if(err== 14) return (char*)"PICO_ERR_EFAULT";
-	    PICO_ERR_ENOMEM = 12,
+		// ...
-	    PICO_ERR_EACCESS = 13,
+		if(err== 16) return (char*)"PICO_ERR_EBUSY";
-	    PICO_ERR_EFAULT = 14,
+		if(err== 17) return (char*)"PICO_ERR_EEXIST";
-	   
+		// ...
-	    PICO_ERR_EBUSY = 16,
+		if(err== 22) return (char*)"PICO_ERR_EINVAL";
-	    PICO_ERR_EEXIST = 17,
+		// ...
-	   
+		if(err== 64) return (char*)"PICO_ERR_ENONET";
-	    PICO_ERR_EINVAL = 22,
+		// ...
+		if(err== 71) return (char*)"PICO_ERR_EPROTO";
+		// ...
+		if(err== 92) return (char*)"PICO_ERR_ENOPROTOOPT";
+		if(err== 93) return (char*)"PICO_ERR_EPROTONOSUPPORT";
+		// ...
+		if(err== 95) return (char*)"PICO_ERR_EOPNOTSUPP";
+		if(err== 98) return (char*)"PICO_ERR_EADDRINUSE";
+		if(err== 99) return (char*)"PICO_ERR_EADDRNOTAVAIL";
+		if(err==100) return (char*)"PICO_ERR_ENETDOWN";
+		if(err==101) return (char*)"PICO_ERR_ENETUNREACH";
+		// ...
+		if(err==104) return (char*)"PICO_ERR_ECONNRESET";
+		// ...
+		if(err==106) return (char*)"PICO_ERR_EISCONN";
+		if(err==107) return (char*)"PICO_ERR_ENOTCONN";
+		if(err==108) return (char*)"PICO_ERR_ESHUTDOWN";
+		// ...
+		if(err==110) return (char*)"PICO_ERR_ETIMEDOUT";
+		if(err==111) return (char*)"PICO_ERR_ECONNREFUSED";
+		if(err==112) return (char*)"PICO_ERR_EHOSTDOWN";
+		if(err==113) return (char*)"PICO_ERR_EHOSTUNREACH";
+		return (char*)"UNKNOWN_ERROR";
+	}
 
-	    PICO_ERR_ENONET = 64,
+/*
 
-	    PICO_ERR_EPROTO = 71,
+#define PICO_SOCKET_STATE_UNDEFINED       0x0000u
+#define PICO_SOCKET_STATE_SHUT_LOCAL      0x0001u
+#define PICO_SOCKET_STATE_SHUT_REMOTE     0x0002u
+#define PICO_SOCKET_STATE_BOUND           0x0004u
+#define PICO_SOCKET_STATE_CONNECTED       0x0008u
+#define PICO_SOCKET_STATE_CLOSING         0x0010u
+#define PICO_SOCKET_STATE_CLOSED          0x0020u
 
-	    PICO_ERR_ENOPROTOOPT = 92,
+# define PICO_SOCKET_STATE_TCP                0xFF00u
-	    PICO_ERR_EPROTONOSUPPORT = 93,
+# define PICO_SOCKET_STATE_TCP_UNDEF          0x00FFu
+# define PICO_SOCKET_STATE_TCP_CLOSED         0x0100u
+# define PICO_SOCKET_STATE_TCP_LISTEN         0x0200u
+# define PICO_SOCKET_STATE_TCP_SYN_SENT       0x0300u
+# define PICO_SOCKET_STATE_TCP_SYN_RECV       0x0400u
+# define PICO_SOCKET_STATE_TCP_ESTABLISHED    0x0500u
+# define PICO_SOCKET_STATE_TCP_CLOSE_WAIT     0x0600u
+# define PICO_SOCKET_STATE_TCP_LAST_ACK       0x0700u
+# define PICO_SOCKET_STATE_TCP_FIN_WAIT1      0x0800u
+# define PICO_SOCKET_STATE_TCP_FIN_WAIT2      0x0900u
+# define PICO_SOCKET_STATE_TCP_CLOSING        0x0a00u
+# define PICO_SOCKET_STATE_TCP_TIME_WAIT      0x0b00u
+# define PICO_SOCKET_STATE_TCP_ARRAYSIZ       0x0cu
 
-	    PICO_ERR_EOPNOTSUPP = 95,
+*/
-	    PICO_ERR_EADDRINUSE = 98,
+	char *picoTCP::beautify_pico_state(int state)
-	    PICO_ERR_EADDRNOTAVAIL = 99,
+	{
-	    PICO_ERR_ENETDOWN = 100,
+		char state_str[512];
-	    PICO_ERR_ENETUNREACH = 101,
+		char *str_ptr = state_str;
 
-	    PICO_ERR_ECONNRESET = 104,
+		if(state & PICO_SOCKET_STATE_UNDEFINED) {
-
+			sprintf(str_ptr, "UNDEFINED ");
-	    PICO_ERR_EISCONN = 106,
+			str_ptr += strlen("UNDEFINED ");
-	    PICO_ERR_ENOTCONN = 107,
-	    PICO_ERR_ESHUTDOWN = 108,
-
-	    PICO_ERR_ETIMEDOUT = 110,
-	    PICO_ERR_ECONNREFUSED = 111,
-	    PICO_ERR_EHOSTDOWN = 112,
-	    PICO_ERR_EHOSTUNREACH = 113,
 		}
-		return err_text;
+		if(state & PICO_SOCKET_STATE_SHUT_LOCAL) {
-		*/
+			sprintf(str_ptr, "SHUT_LOCAL ");
+			str_ptr += strlen("SHUT_LOCAL ");
+		}
+		if(state & PICO_SOCKET_STATE_SHUT_REMOTE) {
+			sprintf(str_ptr, "SHUT_REMOTE ");
+			str_ptr += strlen("SHUT_REMOTE ");
+		}
+		if(state & PICO_SOCKET_STATE_BOUND) {
+			sprintf(str_ptr, "BOUND ");
+			str_ptr += strlen("BOUND ");
+		}
+		if(state & PICO_SOCKET_STATE_CONNECTED) {
+			sprintf(str_ptr, "CONNECTED ");
+			str_ptr += strlen("CONNECTED ");
+		}
+		if(state & PICO_SOCKET_STATE_CLOSING) {
+			sprintf(str_ptr, "CLOSING ");
+			str_ptr += strlen("CLOSING ");
+		}
+		if(state & PICO_SOCKET_STATE_CLOSED) {
+			sprintf(str_ptr, "CLOSED ");
+			str_ptr += strlen("CLOSED ");
+		}
+
+
+		if(state & PICO_SOCKET_STATE_TCP) {
+			sprintf(str_ptr, "TCP ");
+			str_ptr += strlen("TCP ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_UNDEF) {
+			sprintf(str_ptr, "TCP_UNDEF ");
+			str_ptr += strlen("TCP_UNDEF ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_CLOSED) {
+			sprintf(str_ptr, "TCP_CLOSED ");
+			str_ptr += strlen("TCP_CLOSED ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_LISTEN) {
+			sprintf(str_ptr, "TCP_LISTEN ");
+			str_ptr += strlen("TCP_LISTEN ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_SYN_SENT) {
+			sprintf(str_ptr, "TCP_SYN_SENT ");
+			str_ptr += strlen("TCP_SYN_SENT ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_SYN_RECV) {
+			sprintf(str_ptr, "TCP_SYN_RECV ");
+			str_ptr += strlen("TCP_SYN_RECV ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_ESTABLISHED) {
+			sprintf(str_ptr, "TCP_ESTABLISHED ");
+			str_ptr += strlen("TCP_ESTABLISHED ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_CLOSE_WAIT) {
+			sprintf(str_ptr, "TCP_CLOSE_WAIT ");
+			str_ptr += strlen("TCP_CLOSE_WAIT ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_LAST_ACK) {
+			sprintf(str_ptr, "TCP_LAST_ACK ");
+			str_ptr += strlen("TCP_LAST_ACK ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_FIN_WAIT1) {
+			sprintf(str_ptr, "TCP_FIN_WAIT1 ");
+			str_ptr += strlen("TCP_FIN_WAIT1 ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_FIN_WAIT2) {
+			sprintf(str_ptr, "TCP_FIN_WAIT2 ");
+			str_ptr += strlen("TCP_FIN_WAIT2 ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_CLOSING) {
+			sprintf(str_ptr, "TCP_CLOSING ");
+			str_ptr += strlen("TCP_CLOSING ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_TIME_WAIT) {
+			sprintf(str_ptr, "TCP_TIME_WAIT ");
+			str_ptr += strlen("TCP_TIME_WAIT ");
+		}
+		if(state & PICO_SOCKET_STATE_TCP_ARRAYSIZ) {
+			sprintf(str_ptr, "TCP_ARRAYSIZ ");
+			str_ptr += strlen("TCP_ARRAYSIZ ");
+		}
+		return (char*)state_str;
 	}
 }

src/picoTCP.hpp

@@ -156,6 +156,16 @@ namespace ZeroTier
 		 * Close a Connection - Called from SocketTap
 		 */
 		int pico_Close(Connection *conn);
+
+    	/*
+		 * Converts error code to pretty string
+		 */
+		static char *beautify_pico_error(int err);
+
+		/*
+		 *
+		 */
+		static char *beautify_pico_state(int state);
 	};
 }
 

test/echotest.cpp

@@ -59,13 +59,18 @@ std::map<std::string, std::string> testConf;
 
 void loadTestConfigFile(std::string filepath)
 {
-	std::string key;
+	std::string key, value, prefix;
-	std::string value;
 	std::ifstream testFile;
 	testFile.open(filepath.c_str());
 	while (testFile >> key >> value) {
-		if(key[0] != '#')
+		if(key == "name") {
-	    	testConf[key] = value;
+			prefix = value;
+		}
+		if(key[0] != '#' && key[0] != ';') {
+	    	testConf[prefix + "." + key] = value; // format: alice.ipv4 172.30.30.1
+	        //fprintf(stderr, "%s.%s = %s\n", prefix.c_str(), key.c_str(), testConf[prefix + "." + key].c_str());
+	    }
+
 	}
 	testFile.close();
 }
@@ -108,16 +113,16 @@ void start_echo_mode(std::string ipstr, int listen_port)
 			DEBUG_ERROR("error accepting connection (err=%d, errno=%s)", accfd, strerror(errno));
 			return;
 		}
-		DEBUG_TEST("connection accepted! (fd=%d)", accfd);
+		DEBUG_TEST("\n\nconnection accepted! (fd=%d)", accfd);
 
 		// Read initial test parameters from other host
-		int err = 0;
+		int err   = 0;
-		int mode = 0; // rx/tx
+		int mode  = 0; // rx/tx
 		int count = 0; // of incoming byte stream, or requested outgoing
+		int len   = sizeof mode + sizeof count;
+		int tot   = 0; // total bytes read from remote test stream (selftest)
 		char pbuf[64]; // test parameter buffer
 		char rbuf[MAX_RX_BUF_SZ];
-		int len = sizeof mode + sizeof count;
-		int tot = 0; // total bytes read from remote test stream (selftest)
 
 		memset(pbuf, 0, sizeof pbuf);
 
@@ -166,10 +171,10 @@ void start_echo_mode(std::string ipstr, int listen_port)
 				tot += err;
 				totKB = (float)tot / (float)1024;
 				totMB = (float)tot / (float)(1024*1024);
-				//DEBUG_TEST("read = %d, totB = %d, totKB = %3f, totMB = %3f", err, tot, totKB, totMB);
+				DEBUG_TEST("read = %d, totB = %d, totKB = %3f, totMB = %3f", err, tot, totKB, totMB);
 
 			}
-			//DEBUG_TEST("total received = %d (%d MB)", tot);
+			DEBUG_TEST("total received = %d (%d MB)", tot);
 			long int end_time = get_now_ts();	
 			DEBUG_TEST("read last byte (tot=%d). stopping timer. sending test data back to remote selftest", tot);
 
@@ -182,7 +187,7 @@ void start_echo_mode(std::string ipstr, int listen_port)
 				DEBUG_ERROR("error while sending test data to remote selftest host (err=%d, errno=%s)", err, strerror(errno));
 				return;
 			}
-			DEBUG_TEST("sleeping before closing socket and accepting further selftest connections\n\n");
+			DEBUG_TEST("sleeping before closing socket and accepting further selftest connections");
 			sleep(3);
 		}
 
@@ -206,7 +211,7 @@ void start_echo_mode(std::string ipstr, int listen_port)
 				tot += err;
 				totKB = (float)tot / (float)1024;
 				totMB = (float)tot / (float)(1024*1024);
-				//DEBUG_TEST("read = %d, totB = %d, totKB = %3f, totMB = %3f", err, tot, totKB, totMB);
+				DEBUG_TEST("wrote = %d, totB = %d, totKB = %3f, totMB = %3f", err, tot, totKB, totMB);
 			}
 			DEBUG_TEST("sleeping before closing socket and accepting further selftest connections");
 			sleep(3);
@@ -216,14 +221,19 @@ void start_echo_mode(std::string ipstr, int listen_port)
 	close(sockfd);
 }
 
+
 int main(int argc , char *argv[])
 {
-    if(argc < 1) {
+    if(argc < 5) {
-        fprintf(stderr, "usage: echo <alice|bob>.conf\n");
+        fprintf(stderr, "usage: echotest <selftest.conf> <alice|bob|ted|carol> to <bob|alice|ted|carol>\n");
-        fprintf(stderr, " - Define your test environment in *.conf files.\n");     
+        fprintf(stderr, "e.g. : echotest test/selftest.conf bob to alice\n");   
         return 1;
     }
 
+	std::string from = argv[2];
+	std::string   to = argv[4];
+	std::string   me = from;
+
     int start_port       = 0;
     int port_offset      = 0;
 	int echo_listen_port = 0;
@@ -233,29 +243,44 @@ int main(int argc , char *argv[])
 	std::string nwid, stype, path = argv[1];
 	std::string ipstr, ipstr6, local_ipstr, local_ipstr6, remote_ipstr, remote_ipstr6;
 
-	// if a test config file was specified:
+	// loaf config file
-	if(path.find(".conf") != std::string::npos) {
+	if(path.find(".conf") == std::string::npos) {
-		loadTestConfigFile(path);
+		fprintf(stderr, "Possibly invalid conf file. Exiting...\n");
-		start_port = atoi(testConf["start_port"].c_str());
+		exit(0);
-		port_offset = atoi(testConf["port_offset"].c_str());
-		local_ipstr   = testConf["local_ipv4"];
-		local_ipstr6  = testConf["local_ipv6"];
-		local_echo_ipv4 = testConf["local_echo_ipv4"];
-		remote_ipstr  = testConf["remote_ipv4"];
-		remote_ipstr6 = testConf["remote_ipv6"];
-
-		if(strcmp(testConf["name"].c_str(), "alice") == 0)
-			echo_listen_port = start_port+port_offset;
-		else if(strcmp(testConf["name"].c_str(), "bob") == 0)
-			echo_listen_port = start_port+port_offset+1;
-
-		fprintf(stderr, "\tlocal_ipstr     = %s\n", local_ipstr.c_str());
-		fprintf(stderr, "\tremote_ipstr    = %s\n", remote_ipstr.c_str());		
-		fprintf(stderr, "\tstart_port      = %d\n", start_port);
-		fprintf(stderr, "\tport_offset     = %d\n", port_offset);
-		fprintf(stderr, "\tlocal_echo_ipv4 = %s\n", local_echo_ipv4.c_str());
 	}
+	loadTestConfigFile(path);
+
+	// get echo details
+	local_echo_ipv4 = testConf[me + ".echo_ipv4"];
+	nwid = testConf[me + ".nwid"];
+	start_port = atoi(testConf[me + ".port"].c_str());
+	port_offset = 100;
+
+	// get destination details
+	remote_ipstr  = testConf[to + ".ipv4"];
+	remote_ipstr6 = testConf[to + ".ipv6"];
+
+	if(me == "alice" || me == "ted") {
+		echo_listen_port = start_port + port_offset + 1;
+	}
+	if(me == "bob" || me == "carol") {
+		echo_listen_port = start_port + port_offset;
+	}
+
+	fprintf(stderr, "Test Parameters:\n\n");
+
+	fprintf(stderr, "ORIGIN:\n\n");
+	fprintf(stderr, "\tlocal_ipstr      = %s\n", local_ipstr.c_str());
+	fprintf(stderr, "\tlocal_ipstr6     = %s\n", local_ipstr6.c_str());
+	fprintf(stderr, "\tstart_port       = %d\n", start_port);
+	fprintf(stderr, "\tnwid             = %s\n", nwid.c_str());
+	fprintf(stderr, "\ttype             = %s\n\n", stype.c_str());
+
+	fprintf(stderr, "DESTINATION:\n\n");
+	fprintf(stderr, "\tremote_ipstr     = %s\n", remote_ipstr.c_str());
+	fprintf(stderr, "\tremote_ipstr6    = %s\n", remote_ipstr6.c_str());
 	
+	fprintf(stderr, "I am %s\n", me.c_str());
 	DEBUG_TEST("Starting echo mode... %s : %d", local_echo_ipv4.c_str(), echo_listen_port);
 	start_echo_mode(local_echo_ipv4, echo_listen_port);
 	return 1;