Consolidated tcp_write() logic in picoTCP driver, updated tcp_closure/tcp_shutdown logic in picoTCP driver, stubbed out more doxygen comments

include/Debug.hpp

@@ -33,7 +33,7 @@
 #define ZT_MSG_INFO        true // Information which is generally useful to any developer
 #define ZT_MSG_TEST        true // For use in selftest
 #define ZT_MSG_TRANSFER    true // RX/TX specific statements
-#define ZT_MSG_EXTRA       true // If nothing in your world makes sense
+#define ZT_MSG_EXTRA       false // If nothing in your world makes sense
 
 #define ZT_COLOR           true
 

src/ExampleStackDriver.cpp

@@ -0,0 +1,37 @@
+/*
+ * ZeroTier SDK - Network Virtualization Everywhere
+ * Copyright (C) 2011-2017  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+
+/*
+
+This file is only meant for those who wish to replace the currently supported network stacks (lwIP and picoTCP) with
+one of their own choosing. If you're looking for information on how to integrate libzt with your application, you 
+should instead go view the (examples) folder. Now let us get started.
+
+ TODO
+
+
+*/

src/VirtualSocket.hpp

@@ -45,12 +45,46 @@
 #include "VirtualTap.hpp"
 #include "RingBuffer.hpp"
 
-#define VS_OK                         100
+#define VS_STATE_INACTIVE             0x000000u // Default value for newly created VirtualSocket
-#define VS_SHOULD_STOP                101
+#define VS_STATE_ACTIVE               0x000001u // VirtualSocket is RX'ing or TX'ing without issue 
-#define VS_STOPPED                    102
+#define VS_STATE_SHOULD_SHUTDOWN      0x000002u // Application, stack driver, or stack marked this VirtualSocket for death
-#define VS_STATE_UNHANDLED_CONNECTED  103
+#define VS_STATE_SHUTDOWN             0x000004u // VirtualSocket and underlying protocol control structures will not RX/TX
-#define VS_STATE_CONNECTED            104
+#define VS_STATE_CLOSED               0x000008u // VirtualSocket and underlying protocol control structures are closed
-#define VS_STATE_LISTENING            105
+#define VS_STATE_UNHANDLED_CONNECTED  0x000010u // stack callback has received a connection but we haven't dealt with it
+#define VS_STATE_CONNECTED            0x000020u // stack driver has akwnowledged new connection
+#define VS_STATE_LISTENING            0x000040u // virtual socket is listening for incoming connections
+
+#define VS_OPT_TCP_NODELAY            0x000000u // Nagle's algorithm
+#define VS_OPT_SO_LINGER              0x000001u // VirtualSocket waits for data transmission before closure
+/*
+#define VS_RESERVED                   0x000002u // 
+#define VS_RESERVED                   0x000004u //
+#define VS_RESERVED                   0x000008u // 
+#define VS_RESERVED                   0x000010u // 
+#define VS_RESERVED                   0x000020u //
+#define VS_RESERVED                   0x000040u //
+*/
+#define VS_OPT_FD_NONBLOCKING         0x000080u // Whether the VirtualSocket exhibits non-blocking behaviour
+/*
+#define VS_RESERVED                   0x000100u //
+#define VS_RESERVED                   0x000200u //
+#define VS_RESERVED                   0x000400u //
+#define VS_RESERVED                   0x000800u //
+#define VS_RESERVED                   0x001000u //
+#define VS_RESERVED                   0x002000u //
+#define VS_RESERVED                   0x004000u //
+#define VS_RESERVED                   0x008000u //
+#define VS_RESERVED                   0x010000u //
+#define VS_RESERVED                   0x020000u //
+#define VS_RESERVED                   0x040000u //
+#define VS_RESERVED                   0x080000u //
+#define VS_RESERVED                   0x100000u //
+#define VS_RESERVED                   0x200000u //
+#define VS_RESERVED                   0x400000u // 
+#define VS_RESERVED                   0x800000u // 
+*/
+
+#define vs_is_nonblocking(vs)  (((vs)->optflags & VS_OPT_FD_NONBLOCKING) != 0)
 
 namespace ZeroTier {
 
@@ -58,20 +92,34 @@ namespace ZeroTier {
 
 	/**
 	 * An abstraction of a socket that operates between the application-exposed platform-sockets 
-	 * and the network stack's representation of a protocol control block. This object is used by
+	 * and the network stack's representation of a protocol control structure. This object is used by
 	 * the POSIX socket emulation layer and stack drivers.
 	 */
 	class VirtualSocket
 	{
 	private:
-		int _state = VS_OK;
+		int _state = VS_STATE_INACTIVE;
 	public:
 		RingBuffer<unsigned char> *TXbuf;
 		RingBuffer<unsigned char> *RXbuf;
 		Mutex _tx_m, _rx_m, _op_m;
 		PhySocket *sock = NULL;
 
-		// State control
+		/**
+		 * Sets the VirtualSocket's state value
+		 */
+		void apply_state(int state) {
+			// states may be set by application or by stack callbacks, thus this must be guarded
+			_op_m.lock();
+			_state &= state; 
+#if defined (STACK_PICO)
+			DEBUG_EXTRA("APPLY STATE=%d (state=%d, vs=%p, ps=%p)", _state, state, this, picosock);
+#endif
+#if defined (STACK_LWIP)
+			DEBUG_EXTRA("APPLY STATE=%d (state=%d, vs=%p, pcb=%p)", _state, state, this, pcb);
+#endif
+			_op_m.unlock();
+		}
 		/**
 		 * Sets the VirtualSocket's state value
 		 */
@@ -79,20 +127,32 @@ namespace ZeroTier {
 			// states may be set by application or by stack callbacks, thus this must be guarded
 			_op_m.lock();
 			_state = state; 
-			//DEBUG_EXTRA("SET STATE = %d (vs=%p)", _state, this);
+#if defined (STACK_PICO)
+			DEBUG_EXTRA("SET STATE=%d (state=%d, vs=%p, ps=%p)", _state, state, this, picosock);
+#endif
+#if defined (STACK_LWIP)
+			DEBUG_EXTRA("SET STATE=%d (state=%d, vs=%p, pcb=%p)", _state, state, this, pcb);
+#endif			
 			_op_m.unlock();
 		}
 		/**
 		 * Gets the VirtualSocket's state value
 		 */
 		int get_state() {
-			//DEBUG_EXTRA("GET STATE = %d (vs=%p)", _state, this);
+#if defined (STACK_PICO)
+			DEBUG_EXTRA("GET STATE=%d (vs=%p, ps=%p)", _state, this, picosock);
+#endif
+#if defined (STACK_LWIP)
+			DEBUG_EXTRA("GET STATE=%d (vs=%p, pcb=%p)", _state, this, pcb);
+#endif			
 			return _state; 
 		}
 #if defined(STACK_PICO)
 		struct pico_socket *picosock = NULL;
 #endif
 #if defined(STACK_LWIP)
+		int32_t optflags = 0;
+		int linger;
 		void *pcb = NULL; // Protocol Control Block
 		/*
 		  - TCP_WRITE_FLAG_COPY: indicates whether the new memory should be allocated
@@ -119,10 +179,9 @@ namespace ZeroTier {
 
 		std::queue<VirtualSocket*> _AcceptedConnections;
 		VirtualTap *tap = NULL;
-		std::time_t closure_ts = 0;
 
 		VirtualSocket() {
-
+			DEBUG_EXTRA("this=%p",this);
 			memset(&local_addr, 0, sizeof(sockaddr_storage));
 			memset(&peer_addr,  0, sizeof(sockaddr_storage));
 
@@ -131,7 +190,6 @@ namespace ZeroTier {
 			RXbuf = new RingBuffer<unsigned char>(ZT_TCP_RX_BUF_SZ);
 
 			// socketpair, I/O channel between app and stack drivers
-			closure_ts = -1;
 			ZT_PHY_SOCKFD_TYPE fdpair[2];
 			if (socketpair(PF_LOCAL, SOCK_STREAM, 0, fdpair) < 0) {
 				if (errno < 0) {
@@ -149,11 +207,19 @@ namespace ZeroTier {
 			}
 		}
 		~VirtualSocket() {
+			DEBUG_EXTRA("this=%p",this);
 			close(app_fd);
 			close(sdk_fd);
 			delete TXbuf;
 			delete RXbuf;
 			TXbuf = RXbuf = NULL;
+#if defined(STACK_PICO)
+			picosock->priv = NULL;
+			picosock = NULL;
+#endif			
+#if defined(STACK_LWIP)
+			pcb = NULL;
+#endif
 		}
 	};
 

src/VirtualTap.cpp

@@ -296,18 +296,32 @@ namespace ZeroTier {
 
 	void VirtualTap::phyOnUnixClose(PhySocket *sock, void **uptr)
 	{
+		DEBUG_EXTRA();
+		/*
+		int err = 0;
 		if (sock) {
 			VirtualSocket *vs = (VirtualSocket*)uptr;
 			if (vs) {
-				Close(vs);
+				if (vs->get_state() != VS_STATE_CLOSED && vs->get_state() != VS_STATE_LISTENING) {
+					DEBUG_EXTRA("vs=%p, vs->get_state()=%d, vs->picosock->state=%d", vs, vs->get_state(), vs->picosock->state);
+					// doesn't make sense to shut down a listening socket, just close it
+					if ((err = vs->tap->Shutdown(vs, SHUT_RDWR)) < 0) {
+						DEBUG_ERROR("error while shutting down socket");
+						handle_general_failure();
+						return;
+					}
+				}
+				//picostack->pico_Close(vs);
 			}
 		} else {
 			handle_general_failure();
 		}
+		*/
 	}
 
 	void VirtualTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data, ssize_t len)
 	{
+		DEBUG_EXTRA();
 		VirtualSocket *vs = (VirtualSocket*)*uptr;
 		if (vs == NULL) {
 			handle_general_failure();
@@ -588,6 +602,7 @@ namespace ZeroTier {
 	// Write data from app socket to the virtual wire, either raw over VL2, or via network stack
 	int VirtualTap::Write(VirtualSocket *vs, void *data, ssize_t len) 
 	{
+		DEBUG_EXTRA("vs=%p, fd=%d, data=%p, len=%d", vs, vs->app_fd, data, len);
 		int err = -1;
 #if defined(NO_STACK)
 #endif
@@ -668,14 +683,29 @@ namespace ZeroTier {
 			handle_general_failure();
 			return -1;
 		}
+		if (vs->sock) {
+			DEBUG_EXTRA("calling _phy.close()");
+			_phy.close(vs->sock, true);
+		}
 		removeVirtualSocket(vs);
 #if defined(STACK_PICO)
-		if (picostack) {
+			if (vs->get_state() != VS_STATE_CLOSED && vs->get_state() != VS_STATE_LISTENING) {
-			err = picostack->pico_Close(vs);
+				DEBUG_EXTRA("vs=%p, vs->get_state()=%d, vs->picosock->state=%d", vs, vs->get_state(), vs->picosock->state);
-		} else {
+				// doesn't make sense to shut down a listening socket, just close it
-			handle_general_failure();
+				if ((err = vs->tap->Shutdown(vs, SHUT_RDWR)) < 0) {
-			return -1;
+					DEBUG_ERROR("error while shutting down socket");
-		}
+					handle_general_failure();
+					return - 1;
+				}
+			}
+			picostack->pico_Close(vs);
+			removeVirtualSocket(vs);
+			if (vs->socket_type == SOCK_STREAM) {
+				while (!(vs->picosock->state & PICO_SOCKET_STATE_CLOSED)) {
+					nanosleep((const struct timespec[]) {{0, (ZT_ACCEPT_RECHECK_DELAY * 1000000)}}, NULL);
+					DEBUG_EXTRA("virtual lingering on socket, ps=%p, buf remaining=%d",vs->picosock, vs->TXbuf->count());
+				}
+			}
 #endif
 #if defined(STACK_LWIP)
 		if (lwipstack) {
@@ -685,9 +715,6 @@ namespace ZeroTier {
 			return -1;
 		}
 #endif
-		if (vs->sock) {
-			_phy.close(vs->sock, false);
-		}
 		return err;
 	}
 

src/lwIP.cpp

@@ -675,13 +675,13 @@ namespace ZeroTier
 			}
 			else {
 				// place a request for the stack to close this VirtualSocket's PCB
-				vs->set_state(VS_SHOULD_STOP);
+				vs->set_state(VS_STATE_SHOULD_SHUTDOWN);
 				// wait for indication of success, this will block if the PCB can't close
 				while (true) {
 					sleep(1);
 					nanosleep((const struct timespec[]) {{0, (ZT_API_CHECK_INTERVAL * 1000000)}}, NULL);
 					DEBUG_EXTRA("checking closure state... pcb->state=%d", tpcb->state);
-					if (vs->get_state() == VS_STOPPED || tpcb->state == CLOSED) {
+					if (vs->get_state() == VS_STATE_CLOSED || tpcb->state == CLOSED) {
 						return ERR_OK;
 					}
 				}
@@ -689,7 +689,7 @@ namespace ZeroTier
 		}
 		if (vs->socket_type == SOCK_DGRAM) {
 			// place a request for the stack to close this VirtualSocket's PCB
-			vs->set_state(VS_SHOULD_STOP);
+			vs->set_state(VS_STATE_SHOULD_SHUTDOWN);
 		}
 		return err;
 	}
@@ -742,7 +742,7 @@ namespace ZeroTier
 		struct pbuf* q = p;
 		if (p == NULL) {
 			DEBUG_INFO("p=0x0 for pcb=%p, vs->pcb=%p, this indicates a closure. No need to call tcp_close()", PCB, vs->pcb);
-			vs->set_state(VS_SHOULD_STOP);
+			vs->set_state(VS_STATE_SHOULD_SHUTDOWN);
 			return ERR_ABRT;
 		}
 		vs->tap->_tcpconns_m.lock();
@@ -953,9 +953,8 @@ namespace ZeroTier
 			DEBUG_INFO("fd=%d, vs=%p, pcb=%p", vs->app_fd, vs, PCB, vs->pcb);
 		}
 
-
 		// Handle PCB closure requests (set in lwip_Close())
-		if (vs->get_state() == VS_SHOULD_STOP) {
+		if (vs->get_state() == VS_STATE_SHOULD_SHUTDOWN) {
 			DEBUG_EXTRA("closing pcb=%p, fd=%d, vs=%p", PCB, vs->app_fd, vs);
 			int err = 0;
 			errno = 0;
@@ -996,7 +995,7 @@ namespace ZeroTier
 						err = -1;
 					}
 					else {
-						vs->set_state(VS_STOPPED); // success
+						vs->set_state(VS_STATE_CLOSED); // success
 					}
 				}
 			}
@@ -1097,9 +1096,9 @@ namespace ZeroTier
 			/* Lingers on a close() if data is present. */
 			if (optname == SO_LINGER)
 			{
-				// TODO
+				// we do this at the VirtualSocket layer since lwIP's raw API doesn't currently have a way to do this
-				errno = ENOPROTOOPT;
+				vs->optflags &= VS_OPT_SO_LINGER;
-				return -1;
+				return 0;
 			}
 			/* Permits sending of broadcast messages, if this is supported by the protocol. This option takes an int 
 			value. This is a Boolean option. */
@@ -1356,7 +1355,12 @@ namespace ZeroTier
 			}
 			/* If set, disable the Nagle algorithm. */
 			if (optname == TCP_NODELAY) {
-				pcb->flags |= TF_NODELAY; 
+				int enable_nagle = *((const int*)optval);
+				if (enable_nagle == true) {
+					tcp_nagle_enable(pcb);
+				} else {
+					tcp_nagle_disable(pcb);
+				}
 				return 0;
 			}
 			/* Enable quickack mode if set or disable quickack mode if cleared. */
@@ -1436,12 +1440,13 @@ namespace ZeroTier
 				errno = ENOPROTOOPT;
 				return -1;
 			}
-			/* Lingers on a close() if data is present. */
+			/* Get SO_LINGER flag value */
 			if (optname == SO_LINGER)
 			{
-				// TODO
+				// we do this at the VirtualSocket layer since lwIP's raw API doesn't currently have a way to do this
-				errno = ENOPROTOOPT;
+				optval_tmp = (vs->optflags & VS_OPT_SO_LINGER);
-				return -1;
+				memcpy(optval, &optval_tmp, *optlen);
+				return 0;
 			}
 			/* Permits sending of broadcast messages, if this is supported by the protocol. This option takes an int 
 			value. This is a Boolean option. */
@@ -1695,9 +1700,9 @@ namespace ZeroTier
 				errno = ENOPROTOOPT;
 				err = -1;
 			}
-			/* If set, disable the Nagle algorithm. */
+			/* Get value of Nagle algorithm flag */
 			if (optname == TCP_NODELAY) {
-				optval_tmp = pcb->flags & TF_NODELAY;
+				optval_tmp = tcp_nagle_disabled(pcb);
 				memcpy(optval, &optval_tmp, *optlen);
 				err = 0;
 			}

src/lwIP.hpp

@@ -193,106 +193,109 @@ namespace ZeroTier {
 	class VirtualTap;
 	class VirtualSocket;
 
+	/**
+	 * lwIP network stack driver class
+	 */
 	class lwIP
 	{
 	public:
 
-		/*
+		/**
 		 * Set up an interface in the network stack for the VirtualTap
 		 */
 		void lwip_init_interface(VirtualTap *tap, const InetAddress &ip);
 
-		/*
+		/**
 		 * Returns the number of TCP PCBs currently allocated
 		 */
 		static int lwip_num_current_tcp_pcbs();
 
-		/*
+		/**
 		 * Returns the number of UDP PCBs currently allocated
 		 */
 		static int lwip_num_current_udp_pcbs();
 
-		/*
+		/**
 		 * Returns the number of RAW PCBs currently allocated
 		 */
 		static int lwip_num_current_raw_pcbs();
 
-		/*
+		/**
 		 * Returns the total number of PCBs of any time or state
- 		 */
+		 */
 		int lwip_num_total_pcbs();
 
-		/*
+		/**
 		 * Registers a DNS nameserver with the network stack
 		 */
 		int lwip_add_dns_nameserver(struct sockaddr *addr);
 
-		/*
+		/**
 		 * Un-registers a DNS nameserver from the network stack
 		 */
 		int lwip_del_dns_nameserver(struct sockaddr *addr);
 
-		/*
+		/**
 		 * Main stack loop
 		 */
 		void lwip_loop(VirtualTap *tap);
 
-		/*
+		/**
 		 * Packets from the ZeroTier virtual wire enter the stack here
 		 */
 		void lwip_eth_rx(VirtualTap *tap, const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
 
-		/*
+		/**
 		 * Creates a stack-specific "socket" or "VirtualSocket object"
 		 */
 		int lwip_Socket(void **pcb, int socket_family, int socket_type, int protocol);
 
-		/*
+		/**
 		 * Connect to remote host via userspace network stack interface - Called from VirtualTap
 		 */
 		int lwip_Connect(VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
 
-		/*
+		/**
 		 * Bind to a userspace network stack interface - Called from VirtualTap
 		 */
 		int lwip_Bind(VirtualTap *tap, VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
 
-		/*
+		/**
 		 * Listen for incoming VirtualSockets - Called from VirtualTap
 		 */
 		int lwip_Listen(VirtualSocket *vs, int backlog);
 
-		/*
+		/**
 		 * Accept an incoming VirtualSocket - Called from VirtualTap
 		 */
 		VirtualSocket* lwip_Accept(VirtualSocket *vs);
 
-		/*
+		/**
 		 * Read from RX buffer to application - Called from VirtualTap
 		 */
 		int lwip_Read(VirtualSocket *vs, bool lwip_invoked);
 
-		/*
+		/**
 		 * Write to userspace network stack - Called from VirtualTap
 		 */
 		int lwip_Write(VirtualSocket *vs, void *data, ssize_t len);
 
-		/*
+		/**
 		 * Close a VirtualSocket - Called from VirtualTap
 		 */
 		int lwip_Close(VirtualSocket *vs);
 
-		/*
+		/**
 		 * Shuts down some aspect of a VirtualSocket - Called from VirtualTap
 		 */
 		int lwip_Shutdown(VirtualSocket *vs, int how);
 
-		/*
+		/**
 		 *  Sets a property of a socket
 		 */
 		static int lwip_setsockopt(VirtualSocket *vs, int level, int optname, const void *optval, socklen_t optlen);
 
-		/*
+		/**
 		 *  Gets a property of a socket
 		 */
 		static int lwip_getsockopt(VirtualSocket *vs, int level, int optname, void *optval, socklen_t *optlen);
@@ -301,37 +304,37 @@ namespace ZeroTier {
 
 		//static void netif_status_callback(struct netif *nif);
 
-		/*
+		/**
 		 * Callback for handling received UDP packets (already processed by network stack)
 		 */
 		static err_t lwip_cb_tcp_recved(void *arg, struct tcp_pcb *PCB, struct pbuf *p, err_t err);
 
-		/*
+		/**
 		 * Callback for handling accepted connection
 		 */
 		static err_t lwip_cb_accept(void *arg, struct tcp_pcb *newPCB, err_t err);
 
-		/*
+		/**
 		 * Callback for handling received TCP packets (already processed by stack)
 		 */
 		static void lwip_cb_udp_recved(void * arg, struct udp_pcb * upcb, struct pbuf * p, const ip_addr_t * addr, u16_t port);
 
-		/*
+		/**
 		 * Callback for handling errors from within the network stack
 		 */
 		static void lwip_cb_err(void *arg, err_t err);
 
-		/*
+		/**
 		 * Callback for handling periodic background tasks
 		 */
 		static err_t lwip_cb_poll(void* arg, struct tcp_pcb *PCB);
 
-		/*
+		/**
 		 * Callback for handling confirmation of sent packets
 		 */
 		static err_t lwip_cb_sent(void *arg, struct tcp_pcb *PCB, u16_t len);
 
-		/*
+		/**
 		 * Callback for handling successful connections
 		 */
 		static err_t lwip_cb_connected(void *arg, struct tcp_pcb *PCB, err_t err);

src/picoTCP.cpp

@@ -236,9 +236,7 @@ namespace ZeroTier {
 		while (tap->_run)
 		{
 			tap->_phy.poll(ZT_PHY_POLL_INTERVAL);
-			//_picostack_driver_lock.lock();
 			pico_stack_tick();
-			//_picostack_driver_lock.unlock();
 			tap->Housekeeping();
 		}
 	}
@@ -375,40 +373,15 @@ namespace ZeroTier {
 	{
 		VirtualSocket *vs = (VirtualSocket*)(((VirtualBindingPair*)s->priv)->vs);
 		if (vs == NULL) {
-			DEBUG_ERROR("s->priv yielded no valid vs");
+			DEBUG_EXTRA("vs == NULL");
-			handle_general_failure();
 			return;
 		}
-		Mutex::Lock _l(vs->_tx_m);
+		if (vs->picosock != s) {
-		if (vs == NULL) {
+			DEBUG_ERROR("vs->picosock != s, bad callback");
-			DEBUG_ERROR("invalid VirtualSocket");
-			handle_general_failure();
 			return;
 		}
-		int txsz = vs->TXbuf->count();
+		// we will get the vs->TXBuf->get_buf() reference from within pico_Write
-		if (txsz <= 0)
+		picostack->pico_Write(vs, NULL, vs->TXbuf->count()); 
-			return;
-		//DEBUG_INFO("TXbuf->count()=%d", vs->TXbuf->count());
-
-		int r, max_write_len = std::min(std::min(txsz, ZT_SDK_MTU),ZT_STACK_SOCKET_WR_MAX);
-		if ((r = pico_socket_write(vs->picosock, vs->TXbuf->get_buf(), max_write_len)) < 0) {
-			DEBUG_ERROR("unable to write to pico_socket=%p, err=%d, pico_err=%d, %s",
-				vs->picosock, r, pico_err, beautify_pico_error(pico_err));
-			handle_general_failure();
-			return;
-		}
-		if (vs->socket_type == SOCK_STREAM) {
-			DEBUG_TRANS("len=%5d buf_len=%13d [VSTXBF        -->     NSPICO] proto=0x%04x (TCP)", r, vs->TXbuf->count(), PICO_PROTO_TCP);
-		}
-		if (r == 0) {
-			// DEBUG_ERROR("err=%d, pico_err=%d, %s", r, pico_err, beautify_pico_error(pico_err));
-			// This is a peciliarity of the picoTCP network stack, if we receive no error code, and the size of
-			// the byte stream written is 0, this is an indication that the buffer for this pico_socket is too small
-			// DEBUG_ERROR("pico_socket buffer is too small (adjust ZT_STACK_SOCKET_TX_SZ, ZT_STACK_SOCKET_RX_SZ)");
-			// handle_general_failure();
-		}
-		if (r>0)
-			vs->TXbuf->consume(r);
 	}
 
 	void picoTCP::pico_cb_socket_ev(uint16_t ev, struct pico_socket *s)
@@ -423,7 +396,6 @@ namespace ZeroTier {
 		if (ev & PICO_SOCK_EV_FIN) {
 			DEBUG_EXTRA("PICO_SOCK_EV_FIN (socket closed), picosock=%p", s);
 			//DEBUG_EXTRA("PICO_SOCK_EV_FIN (socket closed), picosock=%p, vs=%p, app_fd=%d, sdk_fd=%d", s, vs, vs->app_fd, vs->sdk_fd);
-			//vs->closure_ts = std::time(nullptr);
 		}
 
 		// PICO_SOCK_EV_ERR - triggered when an error occurs.
@@ -439,26 +411,30 @@ namespace ZeroTier {
 		// allowed to send new data until a local shutdown or close is initiated. PicoTCP is able to
 		// keep the VirtualSocket half-open (only for sending) after the FIN packet has been received,
 		// allowing new data to be sent in the TCP CLOSE WAIT state.
-		if (ev & PICO_SOCK_EV_CLOSE) {
-			if ((err = pico_socket_close(s)) < 0) {
-				DEBUG_ERROR("pico_socket_close()=%d, pico_err=%d, %s", err, pico_err, beautify_pico_error(pico_err));
-			}
-			DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err=%d (%s), picosock=%p", pico_err, beautify_pico_error(pico_err), s);
-			//DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err=%d, picosock=%p, vs=%p, app_fd=%d, sdk_fd=%d", err, s, vs, vs->app_fd, vs->sdk_fd);
-			//vs->closure_ts = std::time(nullptr);
-			return;
-		}
-
-		// --- handle non-error events ---
 
 		VirtualBindingPair *vbp = (VirtualBindingPair*)(s->priv);
 		if (vbp == NULL) {
-			DEBUG_ERROR("s->priv yielded no valid VirtualBindingPair");
+			DEBUG_ERROR("s->priv yielded no valid vbp");
 			handle_general_failure();
 			return;
 		}
 		VirtualTap *tap = static_cast<VirtualTap*>(vbp->tap);
 		VirtualSocket *vs = static_cast<VirtualSocket*>(vbp->vs);
+
+		if (ev & PICO_SOCK_EV_CLOSE) {
+			vs->set_state(VS_STATE_CLOSED);
+			if ((err = pico_socket_shutdown(s, PICO_SHUT_RDWR)) < 0) {
+				DEBUG_ERROR("error while shutting down socket");
+			}
+			if ((err = pico_socket_close(s)) < 0) {
+				DEBUG_ERROR("pico_socket_close()=%d, pico_err=%d, %s", err, pico_err, beautify_pico_error(pico_err));
+			}
+			DEBUG_EXTRA("PICO_SOCK_EV_CLOSE (socket closure) err=%d (%s), picosock=%p", pico_err, beautify_pico_error(pico_err), s);
+			return;
+		}
+
+		// --- handle non-error events ---
+
 		if (vs == NULL) {
 			DEBUG_ERROR("invalid VirtualSocket");
 			handle_general_failure();
@@ -551,6 +527,7 @@ namespace ZeroTier {
 
 	int pico_eth_tx(struct pico_device *dev, void *buf, int len)
 	{
+		//DEBUG_TRANS();
 		//_picostack_driver_lock.lock();
 		VirtualTap *tap = static_cast<VirtualTap*>(dev->tap);
 		if (tap == NULL) {
@@ -573,9 +550,9 @@ namespace ZeroTier {
 
 			char flagbuf[32];
 			memset(&flagbuf, 0, 32);
+/*
 			struct pico_tcp_hdr *hdr;
 			void * tcp_hdr_ptr;
-
 			if (Utils::ntoh(ethhdr->proto) == 0x86dd) { // tcp, ipv6
 				tcp_hdr_ptr = &ethhdr + PICO_SIZE_ETHHDR + PICO_SIZE_IP4HDR;
 			}
@@ -610,7 +587,7 @@ namespace ZeroTier {
 					}
 				}
 			}
-
+*/
 			DEBUG_TRANS("len=%5d dst=%s [%s TX <-- %s] proto=0x%04x %s %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
 				Utils::ntoh(ethhdr->proto), beautify_eth_proto_nums(Utils::ntoh(ethhdr->proto)), flagbuf);
 		}
@@ -625,6 +602,7 @@ namespace ZeroTier {
 	void picoTCP::pico_eth_rx(VirtualTap *tap, const MAC &from,const MAC &to,unsigned int etherType,
 		const void *data,unsigned int len)
 	{
+		//DEBUG_TRANS();
 		//_picostack_driver_lock.lock();
 		if (tap == NULL) {
 			DEBUG_ERROR("invalid tap");
@@ -648,11 +626,11 @@ namespace ZeroTier {
 			mac.setTo(ethhdr.saddr, 6);
 			mac.toAddress(tap->_nwid).toString(nodeBuf);
 
-			char flagbuf[32];
+			char flagbuf[64];
-			memset(&flagbuf, 0, 32);
+			memset(&flagbuf, 0, 64);
+/*
 			struct pico_tcp_hdr *hdr;
 			void * tcp_hdr_ptr;
-
 			if (etherType == 0x86dd) { // tcp, ipv6
 				tcp_hdr_ptr = &ethhdr + PICO_SIZE_ETHHDR + PICO_SIZE_IP4HDR;
 			}
@@ -686,6 +664,7 @@ namespace ZeroTier {
 					}
 				}
 			}
+*/
 			DEBUG_TRANS("len=%5d src=%s [%s RX --> %s] proto=0x%04x %s %s", len, macBuf, nodeBuf, tap->nodeId().c_str(),
 				etherType, beautify_eth_proto_nums(etherType), flagbuf);
 		}
@@ -773,15 +752,6 @@ namespace ZeroTier {
 						DEBUG_ERROR("unable to set RCVBUF size, err=%d, pico_err=%d, %s",
 							t_err, pico_err, beautify_pico_error(pico_err));
 					}
-					/*
-					if (ZT_SOCK_BEHAVIOR_LINGER) {
-						int linger_time_ms = ZT_SOCK_BEHAVIOR_LINGER_TIME;
-						if ((t_err = pico_socket_setoption(psock, PICO_SOCKET_OPT_LINGER, &linger_time_ms)) < 0) {
-							DEBUG_ERROR("unable to set LINGER, err=%d, pico_err=%d, %s",
-								t_err, pico_err, beautify_pico_error(pico_err));
-						}
-					}
-					*/
 				}
 			}
 			*p = psock;
@@ -912,21 +882,27 @@ namespace ZeroTier {
 	int picoTCP::pico_Write(VirtualSocket *vs, void *data, ssize_t len)
 	{
 		int err = 0;
+		void *src_buf = NULL;
 		// TODO: Add RingBuffer overflow checks
-		// DEBUG_INFO("vs=%p, len=%d", vs, len);
+		DEBUG_EXTRA("vs=%p, fd=%d, data=%p, len=%d", vs, vs->app_fd, data, len);
 		if (vs == NULL) {
 			DEBUG_ERROR("invalid vs");
 			handle_general_failure();
 			return ZT_ERR_GENERAL_FAILURE;
 		}
 		Mutex::Lock _l(vs->_tx_m);
-		if (len <= 0) {
+		if (vs->picosock == NULL) {
-			DEBUG_ERROR("invalid write length (len=%d)", len);
+			DEBUG_ERROR("ps == NULL");
 			handle_general_failure();
 			return -1;
 		}
-		if (vs->picosock->state == PICO_SOCKET_STATE_CLOSED) {
+		if (vs->app_fd <= 0) {
-			DEBUG_ERROR("socket is CLOSED, this wrpico_cb_tcp_writeite() will fail");
+			DEBUG_EXTRA("invalid fd");
+			handle_general_failure();
+			return -1;
+		}
+		if (vs->picosock->state & PICO_SOCKET_STATE_CLOSED) {
+			DEBUG_ERROR("socket is PICO_SOCKET_STATE_CLOSED, this pico_tcp_write() will fail");
 			return -1;
 		}
 		if (vs == NULL) {
@@ -935,6 +911,16 @@ namespace ZeroTier {
 			return -1;
 		}
 		if (vs->socket_type == SOCK_DGRAM) {
+			if (data == NULL) {
+				DEBUG_ERROR("data == NULL");
+				handle_general_failure();
+				return -1;
+			}
+			if (len <= 0) {
+				DEBUG_ERROR("invalid write len=%d for SOCK_DGRAM", len);
+				handle_general_failure();
+				return -1;
+			}
 			int r;
 			if ((r = pico_socket_write(vs->picosock, data, len)) < 0) {
 				DEBUG_ERROR("unable to write to picosock=%p, err=%d, pico_err=%d, %s",
@@ -944,25 +930,43 @@ namespace ZeroTier {
 			else {
 				err = r; // successful write
 			}
+			if (vs->socket_type == SOCK_DGRAM) {
+				DEBUG_TRANS("len=%5d buf_len=N/A           [APPFDS        -->     NSPICO] proto=0x%04x (UDP)", r, PICO_PROTO_TCP);
+			}
 		}
 		if (vs->socket_type == SOCK_STREAM) {
-			int original_txsz = vs->TXbuf->count();
+			if (len > 0 && data != NULL) {
-			if (original_txsz + len >= ZT_TCP_TX_BUF_SZ) {
+				
-				DEBUG_ERROR("txsz=%d, len=%d", original_txsz, len);
+				src_buf = data; // --- Data source: poll loop I/O buffer ---
-				DEBUG_ERROR("TX buffer is too small, try increasing ZT_TCP_TX_BUF_SZ in libzt.h");
+
-				handle_general_failure();
+				// in this case, we've recieved data on the 'data' buffer, add it to TX ringbuffer, then try to handle it from there
-				return ZT_ERR_GENERAL_FAILURE;
+				int original_txsz = vs->TXbuf->count();
-			}
+				if (original_txsz + len >= ZT_TCP_TX_BUF_SZ) {
-			int buf_w = vs->TXbuf->write((const unsigned char*)data, len);
+					DEBUG_ERROR("txsz=%d, len=%d", original_txsz, len);
+					DEBUG_ERROR("TX buffer is too small, try increasing ZT_TCP_TX_BUF_SZ in libzt.h");
+					handle_general_failure();
+					return ZT_ERR_GENERAL_FAILURE;
+				}
+				int buf_w = vs->TXbuf->write((const unsigned char*)data, len);
 				if (buf_w != len) {
-				// because we checked ZT_TCP_TX_BUF_SZ above, this should not happen
+					// because we checked ZT_TCP_TX_BUF_SZ above, this should not happen
-				DEBUG_ERROR("TX wrote only %d but expected to write %d", buf_w, len);
+					DEBUG_ERROR("wrote only len=%d but expected to write len=%d", buf_w, len);
-				handle_general_failure();
+					handle_general_failure();
-				return ZT_ERR_GENERAL_FAILURE;
+					return ZT_ERR_GENERAL_FAILURE;
+				}
+			} else if (len == 0 && data == NULL) {
+				DEBUG_EXTRA("len=0 => write request from poll loop or callback");
+
+				src_buf = vs->TXbuf->get_buf(); // --- Data source: TX ringbuffer ---
+
+				// do nothing, all the data we need is already on the TX ringbuffer
+			} else if (len < 0) {
+				DEBUG_ERROR("invalid write len=%d for SOCK_STREAM", len);
 			}
+
 			int txsz = vs->TXbuf->count();
 			int r, max_write_len = std::min(std::min(txsz, ZT_SDK_MTU),ZT_STACK_SOCKET_WR_MAX);
-			if ((r = pico_socket_write(vs->picosock, vs->TXbuf->get_buf(), max_write_len)) < 0) {
+			if ((r = pico_socket_write(vs->picosock, src_buf, max_write_len)) < 0) {
 				DEBUG_ERROR("unable to write to picosock=%p, err=%d, pico_err=%d, %s",
 					vs->picosock, r, pico_err, beautify_pico_error(pico_err));
 				err = -1;
@@ -972,12 +976,9 @@ namespace ZeroTier {
 			}
 			if (r>0) {
 				vs->TXbuf->consume(r);
-			}
+				if (vs->socket_type == SOCK_STREAM) {
-			if (vs->socket_type == SOCK_STREAM) {
+					DEBUG_TRANS("len=%5d buf_len=%13d [VSTXBF        -->     NSPICO] proto=0x%04x (TCP)", r, vs->TXbuf->count(), PICO_PROTO_TCP);
-				DEBUG_TRANS("len=%5d buf_len=%13d [VSTXBF        -->     NSPICO] proto=0x%04x (TCP)", r, vs->TXbuf->count(), PICO_PROTO_TCP);
+				}
-			}
-			if (vs->socket_type == SOCK_DGRAM) {
-				DEBUG_TRANS("len=%5d buf_len=               [APPFDS        -->     NSPICO] proto=0x%04x (UDP)", r, PICO_PROTO_TCP);
 			}
 		}
 		return err;
@@ -985,21 +986,32 @@ namespace ZeroTier {
 
 	int picoTCP::pico_Close(VirtualSocket *vs)
 	{
+		DEBUG_EXTRA();
 		if (vs == NULL) {
 			DEBUG_ERROR("invalid vs");
 			handle_general_failure();
 			return ZT_ERR_GENERAL_FAILURE;
 		}
+		if (vs->get_state() == VS_STATE_CLOSED) {
+			DEBUG_EXTRA("socket already in VS_STATE_CLOSED state");
+			return 0;
+		}
+		if (vs->picosock == NULL) {
+			DEBUG_EXTRA("ps == NULL");
+			return 0;
+		}
+		if (vs->picosock->state & PICO_SOCKET_STATE_CLOSED) {
+			DEBUG_EXTRA("ps already closed, ps=%p", vs->picosock);
+			return 0;
+		}
 		DEBUG_EXTRA("vs=%p, picosock=%p, fd=%d", vs, vs->picosock, vs->app_fd);
 		if (vs == NULL || vs->picosock == NULL)
 			return ZT_ERR_GENERAL_FAILURE;
 		int err = 0;
 		Mutex::Lock _l(vs->tap->_tcpconns_m);
-		if (vs->closure_ts != -1) // it was closed at some point in the past, it'll work itself out
-			return ZT_ERR_OK;
 		if ((err = pico_socket_close(vs->picosock)) < 0) {
 			errno = pico_err;
-			DEBUG_ERROR("error closing pico_socket, err=%d, pico_err=%s, %s",
+			DEBUG_ERROR("error closing pico_socket, err=%d, pico_err=%d, %s",
 				err, pico_err, beautify_pico_error(pico_err));
 		}
 		return err;
@@ -1007,6 +1019,7 @@ namespace ZeroTier {
 
 	int picoTCP::pico_Shutdown(VirtualSocket *vs, int how)
 	{
+		DEBUG_EXTRA("vs=%p, how=%d", vs, how);
 		int err = 0, mode = 0;
 		if (how == SHUT_RD) {
 			mode = PICO_SHUT_RD;
@@ -1069,9 +1082,12 @@ namespace ZeroTier {
 			/* Lingers on a close() if data is present. */
 			if (optname == SO_LINGER)
 			{
-				// TODO
+				int linger_time_ms = *((const int*)optval);
-				errno = ENOPROTOOPT;
+				if ((err = pico_socket_setoption(vs->picosock, PICO_SOCKET_OPT_LINGER, &linger_time_ms)) < 0) {
-				return -1;
+					DEBUG_ERROR("unable to set LINGER, err=%d, pico_err=%d, %s",
+						err, pico_err, beautify_pico_error(pico_err));
+				}
+				return err;
 			}
 			/* Permits sending of broadcast messages, if this is supported by the protocol. This option takes an int 
 			value. This is a Boolean option. */
@@ -1092,16 +1108,28 @@ namespace ZeroTier {
 			/* Sets send buffer size. This option takes an int value. */
 			if (optname == SO_SNDBUF)
 			{
-				// TODO
+				int no_delay = *((const int*)optval);
-				errno = ENOPROTOOPT;
+				if ((err = pico_socket_setoption(vs->picosock, PICO_SOCKET_OPT_SNDBUF, &no_delay) < 0)) {
-				return -1;
+					if (err == PICO_ERR_EINVAL) {
+						DEBUG_ERROR("error while setting PICO_SOCKET_OPT_SNDBUF");
+						errno = EINVAL;
+						err = -1;
+					}
+				}
+				return err;
 			}
 			/* Sets receive buffer size. This option takes an int value. */
 			if (optname == SO_RCVBUF)
 			{
-				// TODO
+				int no_delay = *((const int*)optval);
-				errno = ENOPROTOOPT;
+				if ((err = pico_socket_setoption(vs->picosock, PICO_SOCKET_OPT_RCVBUF, &no_delay) < 0)) {
-				return -1;
+					if (err == PICO_ERR_EINVAL) {
+						DEBUG_ERROR("error while setting PICO_SOCKET_OPT_RCVBUF");
+						errno = EINVAL;
+						err = -1;
+					}
+				}
+				return err;
 			}
 			/* */ 
 			if (optname == SO_STYLE)
@@ -1188,29 +1216,31 @@ namespace ZeroTier {
 				return -1;
 			}
 			if (optname == IP_MULTICAST_IF) {
-				/*
-				if ((err = pico_socket_getoption(p, PICO_TCP_NODELAY, &optval_tmp)) < 0) {
-					if (err == PICO_ERR_EINVAL) {
-						DEBUG_ERROR("error while disabling Nagle's algorithm");
-						errno = ENOPROTOOPT;
-						err = -1;
-					}
-				}
-				memcpy(optval, &optval_tmp, *optlen);
-				*/
 				// TODO
 				errno = ENOPROTOOPT;
 				return -1;
 			}
 			if (optname == IP_MULTICAST_LOOP) {
-				// TODO
+				int loop = *((const int*)optval);
-				errno = ENOPROTOOPT;
+				if ((err = pico_socket_setoption(vs->picosock, PICO_IP_MULTICAST_LOOP, &loop) < 0)) {
-				return -1;
+					if (err == PICO_ERR_EINVAL) {
+						DEBUG_ERROR("error while setting PICO_IP_MULTICAST_TTL");
+						errno = EINVAL;
+						err = -1;
+					}
+				}
+				return err;
 			}
 			if (optname == IP_MULTICAST_TTL) {
-				// TODO
+				int ttl = *((const int*)optval);
-				errno = ENOPROTOOPT;
+				if ((err = pico_socket_setoption(vs->picosock, PICO_IP_MULTICAST_TTL, &ttl) < 0)) {
-				return -1;
+					if (err == PICO_ERR_EINVAL) {
+						DEBUG_ERROR("error while setting PICO_IP_MULTICAST_TTL");
+						errno = EINVAL;
+						err = -1;
+					}
+				}
+				return err;
 			}
 			if (optname == IP_NODEFRAG) {
 				// TODO
@@ -1339,7 +1369,7 @@ namespace ZeroTier {
 			/* If set, disable the Nagle algorithm. */
 			if (optname == TCP_NODELAY) {
 				int no_delay = *((const int*)optval);
-				if ((err = pico_socket_setoption(p, PICO_TCP_NODELAY, &no_delay) < 0)) {
+				if ((err = pico_socket_setoption(vs->picosock, PICO_TCP_NODELAY, &no_delay) < 0)) {
 					if (err == PICO_ERR_EINVAL) {
 						DEBUG_ERROR("error while disabling Nagle's algorithm");
 						errno = EINVAL;
@@ -1451,16 +1481,26 @@ namespace ZeroTier {
 			/* Sets send buffer size. This option takes an int value. */
 			if (optname == SO_SNDBUF)
 			{
-				// TODO
+				if ((err = pico_socket_getoption(vs->picosock, PICO_SOCKET_OPT_SNDBUF, &optval_tmp)) < 0) {
-				errno = ENOPROTOOPT;
+					if (err == PICO_ERR_EINVAL) {
-				return -1;
+						DEBUG_ERROR("error while getting PICO_SOCKET_OPT_SNDBUF");
+						errno = ENOPROTOOPT;
+						err = -1;
+					}
+				}
+				memcpy(optval, &optval_tmp, *optlen);
 			}
 			/* Sets receive buffer size. This option takes an int value. */
 			if (optname == SO_RCVBUF)
 			{
-				// TODO
+				if ((err = pico_socket_getoption(vs->picosock, PICO_SOCKET_OPT_SNDBUF, &optval_tmp)) < 0) {
-				errno = ENOPROTOOPT;
+					if (err == PICO_ERR_EINVAL) {
-				return -1;
+						DEBUG_ERROR("error while getting PICO_SOCKET_OPT_RCVBUF");
+						errno = ENOPROTOOPT;
+						err = -1;
+					}
+				}
+				memcpy(optval, &optval_tmp, *optlen);
 			}
 			/* */ 
 			if (optname == SO_STYLE)
@@ -1552,14 +1592,24 @@ namespace ZeroTier {
 				return -1;
 			}
 			if (optname == IP_MULTICAST_LOOP) {
-				// TODO
+				if ((err = pico_socket_getoption(vs->picosock, PICO_IP_MULTICAST_LOOP, &optval_tmp)) < 0) {
-				errno = ENOPROTOOPT;
+					if (err == PICO_ERR_EINVAL) {
-				return -1;
+						DEBUG_ERROR("error while getting PICO_IP_MULTICAST_TTL");
+						errno = ENOPROTOOPT;
+						err = -1;
+					}
+				}
+				memcpy(optval, &optval_tmp, *optlen);
 			}
 			if (optname == IP_MULTICAST_TTL) {
-				// TODO
+				if ((err = pico_socket_getoption(vs->picosock, PICO_IP_MULTICAST_TTL, &optval_tmp)) < 0) {
-				errno = ENOPROTOOPT;
+					if (err == PICO_ERR_EINVAL) {
-				return -1;
+						DEBUG_ERROR("error while getting PICO_IP_MULTICAST_TTL");
+						errno = ENOPROTOOPT;
+						err = -1;
+					}
+				}
+				memcpy(optval, &optval_tmp, *optlen);
 			}
 			if (optname == IP_NODEFRAG) {
 				// TODO
@@ -1686,21 +1736,7 @@ namespace ZeroTier {
 			}
 			/* If set, disable the Nagle algorithm. */
 			if (optname == TCP_NODELAY) {
-
+				if ((err = pico_socket_getoption(vs->picosock, PICO_TCP_NODELAY, &optval_tmp)) < 0) {
-				/*
-
-				TODO: 
-
-				PICO TCP NODELAY - Nagle algorithm, value casted to (int *) (0 = disabled, 1 = enabled)
-				• PICO SOCKET OPT RCVBUF - Read current receive buffer size for the socket
-				• PICO SOCKET OPT SNDBUF - Read current receive buffer size for the socket
-				• PICO IP MULTICAST IF - (Not supported) Link multicast datagrams are sent from
-				• PICO IP MULTICAST TTL - TTL (0-255) of multicast datagrams
-				• PICO IP MULTICAST LOOP - Loop back a copy of an outgoing multicast datagram, as long
-				as it is a member of the multicast group, or not
-
-				*/
-				if ((err = pico_socket_getoption(p, PICO_TCP_NODELAY, &optval_tmp)) < 0) {
 					if (err == PICO_ERR_EINVAL) {
 						DEBUG_ERROR("error while disabling Nagle's algorithm");
 						errno = ENOPROTOOPT;

src/picoTCP.hpp

@@ -75,12 +75,12 @@
 
 namespace ZeroTier
 {
-	/*
+	/**
 	 * Send raw frames from the stack to the ZeroTier virtual wire
 	 */
 	int pico_eth_tx(struct pico_device *dev, void *buf, int len);
 
-	/*
+	/**
 	 * Read raw frames from RX frame buffer into the stack
 	 */
 	int pico_eth_poll(struct pico_device *dev, int loop_score);
@@ -88,137 +88,140 @@ namespace ZeroTier
 	class VirtualTap;
 	class VirtualSocket;
 
+	/**
+	 * picoTCP network stack driver class
+	 */
 	class picoTCP
 	{
 	public:
 
-		/*
+		/**
 		 * Set up an interface in the network stack for the VirtualTap
 		 */
 		bool pico_init_interface(ZeroTier::VirtualTap *tap);
 
-		/*
+		/**
 		 * Register an address with the stack
 		 */
 		bool pico_register_address(VirtualTap *tap, const InetAddress &ip);
 
-		/*
+		/**
 		 * Adds a route to the picoTCP device
 		 */
 		bool pico_route_add(VirtualTap *tap, const InetAddress &addr, const InetAddress &nm, const InetAddress &gw, int metric);
 
-		/*
+		/**
 		 * Deletes a route from the picoTCP device
 		 */
 		bool pico_route_del(VirtualTap *tap, const InetAddress &addr, const InetAddress &nm, int metric);
 
-		/*
+		/**
 		 * Registers a DNS nameserver with the network stack
 		 */
 		int pico_add_dns_nameserver(struct sockaddr *addr);
 
-		/*
+		/**
 		 * Un-registers a DNS nameserver from the network stack
 		 */
 		int pico_del_dns_nameserver(struct sockaddr *addr);
 
-		/*
+		/**
 		 * Main stack loop
 		 */
 		void pico_loop(VirtualTap *tap);
 
-		/*
+		/**
 		 * Read bytes from the stack to the RX buffer (prepare to be read by app)
 		 */
 		static void pico_cb_tcp_read(VirtualTap *tap, struct pico_socket *s);
 
-		/*
+		/**
 		 * Read bytes from the stack to the RX buffer (prepare to be read by app)
 		 */
 		static void pico_cb_udp_read(VirtualTap *tap, struct pico_socket *s);
 
-		 /*
+		 /**
 		 * Write bytes from TX buffer to stack (prepare to be sent to ZT virtual wire)
 		 */
 		static void pico_cb_tcp_write(VirtualTap *tap, struct pico_socket *s);
 
-		/*
+		/**
 		 * Write bytes from TX buffer to stack (prepare to be sent to ZT virtual wire)
 		 */
 		static void pico_cb_socket_ev(uint16_t ev, struct pico_socket *s);
 
-		/*
+		/**
 		 * Packets from the ZeroTier virtual wire enter the stack here
 		 */
 		void pico_eth_rx(VirtualTap *tap, const ZeroTier::MAC &from, const ZeroTier::MAC &to, 
 			unsigned int etherType, const void *data, unsigned int len);
 
-		/*
+		/**
 		 * Creates a stack-specific "socket" or "VirtualSocket object"
 		 */
 		int pico_Socket(struct pico_socket **p, int socket_family, int socket_type, int protocol);
 
-		/*
+		/**
 		 * Connect to remote host via userspace network stack interface - Called from VirtualTap
 		 */
 		int pico_Connect(VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
 
-		/*
+		/**
 		 * Bind to a userspace network stack interface - Called from VirtualTap
 		 */
 		int pico_Bind(VirtualSocket *vs, const struct sockaddr *addr, socklen_t addrlen);
 
-		/*
+		/**
 		 * Listen for incoming VirtualSockets - Called from VirtualTap
 		 */
 		int pico_Listen(VirtualSocket *vs, int backlog);
 
-		/*
+		/**
 		 * Accept an incoming VirtualSocket - Called from VirtualTap
 		 */
 		VirtualSocket* pico_Accept(VirtualSocket *vs);
 
-		/*
+		/**
 		 * Read from RX buffer to application - Called from VirtualTap
 		 */
 		int pico_Read(VirtualTap *tap, ZeroTier::PhySocket *sock, VirtualSocket *vs, bool stack_invoked);
 
-		/*
+		/**
 		 * Write to userspace network stack - Called from VirtualTap
 		 */
 		int pico_Write(VirtualSocket *vs, void *data, ssize_t len);
 
-		/*
+		/**
 		 * Close a VirtualSocket - Called from VirtualTap
 		 */
 		int pico_Close(VirtualSocket *vs);
 
-		/*
+		/**
 		 *  Shuts down some aspect of a VirtualSocket - Called from VirtualTap
 		 */
 		int pico_Shutdown(VirtualSocket *vs, int how);
 
-		/*
+		/**
 		 *  Sets a property of a socket
 		 */
 		static int pico_setsockopt(VirtualSocket *vs, int level, int optname, const void *optval, socklen_t optlen);
 
-		/*
+		/**
 		 *  Gets a property of a socket
 		 */
 		static int pico_getsockopt(VirtualSocket *vs, int level, int optname, void *optval, socklen_t *optlen);
 
-		/*
+		/**
 		 * Converts a pico_err to its most closely-related errno, and sets errno
 		 */
 		static int map_pico_err_to_errno(int err);
 
-		/*
+		/**
 		 * Converts picoTCP error codes to pretty string
 		 */
 		static char *beautify_pico_error(int err);
 
-		/*
+		/**
 		 * Converts picoTCP socket states into pretty string
 		 */
 		static char *beautify_pico_state(int state);

test/selftest.cpp

@@ -63,7 +63,7 @@
 #define SLAM_INTERVAL          500000  // microseconds
 
 #define WAIT_FOR_TEST_TO_CONCLUDE         0
-#define WAIT_FOR_TRANSMISSION_TO_COMPLETE 5
+#define ARTIFICIAL_SOCKET_LINGER          1
 
 #define STR_SIZE               32
 
@@ -130,6 +130,19 @@
 
 // If running a native instance to test against, use system calls
 #if defined(__NATIVETEST__)
+inline unsigned int gettid()
+{
+#ifdef _WIN32
+    return GetCurrentThreadId();
+#elif defined(__unix__)
+    return static_cast<unsigned int>(::syscall(__NR_gettid));
+#elif defined(__APPLE__)
+    uint64_t tid64;
+    pthread_threadid_np(NULL, &tid64);
+    return static_cast<unsigned int>(tid64);
+#endif
+}
+
 #define SOCKET socket
 #define BIND bind
 #define LISTEN listen
@@ -334,7 +347,7 @@ void tcp_client_4(TCP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "tcp_client_4";
 	std::string msg = "tcp_cs_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "connect to remote host with IPv4 address, write string, read string, compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -367,6 +380,7 @@ void tcp_client_4(TCP_UNIT_TEST_SIG_4)
 	r = READ(fd, rbuf, len);
 	DEBUG_TEST("Sent     : %s", msg.c_str());
 	DEBUG_TEST("Received : %s", rbuf);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 	*passed = (w == len && r == len && !err) && !strcmp(rbuf, msg.c_str());
@@ -381,7 +395,7 @@ void tcp_server_4(TCP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "tcp_server_4";
 	std::string msg = "tcp_cs_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());	
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "accept connection with IPv4 address, read string, write string, compare.\n");
 	int w=0, r=0, fd, client_fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -426,6 +440,7 @@ void tcp_server_4(TCP_UNIT_TEST_SIG_4)
 	r = READ(client_fd, rbuf, len);
 	w = WRITE(client_fd, rbuf, len);
 	DEBUG_TEST("Received : %s, r=%d, w=%d", rbuf, r, w);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	err = CLOSE(client_fd);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
@@ -441,7 +456,7 @@ void tcp_client_6(TCP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "tcp_client_6";
 	std::string msg = "tcp_cs_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "connect to remote host with IPv6 address, write string, read string, compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -473,6 +488,7 @@ void tcp_client_6(TCP_UNIT_TEST_SIG_6)
 
 	w = WRITE(fd, msg.c_str(), len);
 	r = READ(fd, rbuf, len);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 	DEBUG_TEST("Sent     : %s", msg.c_str());
@@ -489,7 +505,7 @@ void tcp_server_6(TCP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "tcp_server_6";
 	std::string msg = "tcp_cs_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());	
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "accept connection with IPv6 address, read string, write string, compare.\n");
 	int w=0, r=0, fd, client_fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -537,6 +553,7 @@ void tcp_server_6(TCP_UNIT_TEST_SIG_6)
 	r = READ(client_fd, rbuf, sizeof rbuf);
 	w = WRITE(client_fd, rbuf, len);
 	DEBUG_TEST("Received : %s", rbuf);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	err = CLOSE(client_fd);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
@@ -554,7 +571,7 @@ void udp_client_4(UDP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "udp_client_4";
 	std::string msg = "udp_cs_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv4 address, send string until response is seen. compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -590,6 +607,7 @@ void udp_client_4(UDP_UNIT_TEST_SIG_4)
 		// rx
 		r = RECVFROM(fd, rbuf, STR_SIZE, 0, (struct sockaddr *)&saddr, (socklen_t *)&serverlen);
 		if (r == strlen(msg.c_str())) {
+			sleep(ARTIFICIAL_SOCKET_LINGER);
 			err = CLOSE(fd);
 			DEBUG_TEST("%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 			sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
@@ -610,7 +628,7 @@ void udp_server_4(UDP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "udp_server_4";
 	std::string msg = "udp_cs_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());	
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv4 address, read single string, send many responses. compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -654,6 +672,7 @@ void udp_server_4(UDP_UNIT_TEST_SIG_4)
 			break;
 		}
 	}
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	DEBUG_TEST("%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
@@ -671,7 +690,7 @@ void udp_client_6(UDP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "udp_client_6";
 	std::string msg = "udp_cs_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv6 address, send string until response is seen. compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -711,7 +730,7 @@ void udp_client_6(UDP_UNIT_TEST_SIG_6)
 		r = RECVFROM(fd, rbuf, len, 0, (struct sockaddr *)&saddr, (socklen_t *)&serverlen);
 		if (r == len) {
 			DEBUG_TEST("[2] complete");
-			sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+			sleep(ARTIFICIAL_SOCKET_LINGER);
 			err = CLOSE(fd);
 			DEBUG_TEST("%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 			sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
@@ -731,7 +750,7 @@ void udp_server_6(UDP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "udp_server_6";
 	std::string msg = "udp_cs_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv6 address, read single string, send many responses. compare.\n");
 	int r, w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -779,8 +798,8 @@ void udp_server_6(UDP_UNIT_TEST_SIG_6)
 			break;
 		}
 	}
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
-	//err = CLOSE(fd);
+	err = CLOSE(fd);
 	DEBUG_TEST("[3/3] complete, %s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 	sprintf(details, "%s, err=%d, r=%d, w=%d", testname.c_str(), err, r, w);
 	DEBUG_TEST("Sent     : %s", msg.c_str());
@@ -805,7 +824,7 @@ void tcp_client_sustained_4(TCP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "tcp_client_sustained_4";
 	std::string msg = "tcp_sustained_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "connect to remote host with IPv4 address, exchange a sequence of packets, check order.\n");
 	int n=0, w=0, r=0, fd, err;	
 	char *rxbuf = (char*)malloc(cnt*sizeof(char));
@@ -831,7 +850,9 @@ void tcp_client_sustained_4(TCP_UNIT_TEST_SIG_4)
 		while (wrem) {
 			int next_write = std::min(4096, wrem);
 			signal(SIGPIPE, SIG_IGN);
+			DEBUG_TEST("writing...");
 			n = WRITE(fd, &txbuf[w], next_write);
+			DEBUG_TEST("wrote=%d", n);
 			if (n > 0) {
 				w += n;
 				wrem -= n;
@@ -856,7 +877,7 @@ void tcp_client_sustained_4(TCP_UNIT_TEST_SIG_4)
 		}
 		long int rx_tf = get_now_ts();	
 		DEBUG_TEST("read=%d", r);
-		sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+		sleep(ARTIFICIAL_SOCKET_LINGER);
 		err = CLOSE(fd);
 		// Compare RX and TX buffer and detect mismatches
 		bool match = true;
@@ -887,7 +908,7 @@ void tcp_client_sustained_6(TCP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "tcp_client_sustained_6";
 	std::string msg = "tcp_sustained_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "connect to remote host with IPv6 address, exchange a sequence of packets, check order.\n");
 	int n=0, w=0, r=0, fd, err;	
 	char *rxbuf = (char*)malloc(cnt*sizeof(char));
@@ -936,7 +957,7 @@ void tcp_client_sustained_6(TCP_UNIT_TEST_SIG_6)
 		}
 		long int rx_tf = get_now_ts();	
 		DEBUG_TEST("read=%d", r);
-		sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+		sleep(ARTIFICIAL_SOCKET_LINGER);
 		err = CLOSE(fd);
 		// Compare RX and TX buffer and detect mismatches
 		bool match = true;
@@ -967,7 +988,7 @@ void tcp_server_sustained_4(TCP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "tcp_server_sustained_4";
 	std::string msg = "tcp_sustained_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "accept connection from host with IPv4 address, exchange a sequence of packets, check order.\n");
 	int n=0, w=0, r=0, fd, client_fd, err;
 	char *rxbuf = (char*)malloc(cnt*sizeof(char));
@@ -1028,6 +1049,7 @@ void tcp_server_sustained_4(TCP_UNIT_TEST_SIG_4)
 		}
 		long int tx_tf = get_now_ts();	
 		DEBUG_TEST("wrote=%d", w);
+		sleep(ARTIFICIAL_SOCKET_LINGER);
 		err = CLOSE(fd);
 		err = CLOSE(client_fd);
 		// Compute time deltas and transfer rates
@@ -1037,7 +1059,6 @@ void tcp_server_sustained_4(TCP_UNIT_TEST_SIG_4)
 		float rx_rate = (float)cnt / (float)rx_dt;
 		sprintf(details, "%s, n=%d, tx_dt=%.2f, rx_dt=%.2f, r=%d, w=%d, tx_rate=%.2f MB/s, rx_rate=%.2f MB/s", 
 			testname.c_str(), cnt, tx_dt, rx_dt, r, w, (tx_rate / float(ONE_MEGABYTE) ), (rx_rate / float(ONE_MEGABYTE) ));
-
 		*passed = (r == cnt && w == cnt && err>=0);
 	}
 	free(rxbuf);
@@ -1051,7 +1072,7 @@ void tcp_server_sustained_6(TCP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "tcp_server_sustained_6";
 	std::string msg = "tcp_sustained_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "accept connection from host with IPv6 address, exchange a sequence of packets, check order.\n");
 	int n=0, w=0, r=0, fd, client_fd, err;
 	char *rxbuf = (char*)malloc(cnt*sizeof(char));
@@ -1115,6 +1136,7 @@ void tcp_server_sustained_6(TCP_UNIT_TEST_SIG_6)
 		}
 		long int tx_tf = get_now_ts();	
 		DEBUG_TEST("wrote=%d", w);
+		sleep(ARTIFICIAL_SOCKET_LINGER);
 		err = CLOSE(fd);
 		err = CLOSE(client_fd);
 		// Compute time deltas and transfer rates
@@ -1138,7 +1160,7 @@ void udp_client_sustained_4(UDP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "udp_client_sustained_4";
 	std::string msg = "udp_sustained_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv4 address, TX n-datagrams\n");
 	int w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -1169,7 +1191,7 @@ void udp_client_sustained_4(UDP_UNIT_TEST_SIG_4)
 			DEBUG_ERROR("error sending packet, err=%d", errno);
 		}
 	}
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	DEBUG_TEST("%s, n=%d, err=%d, w=%d", testname.c_str(), cnt, err, w);
 	sprintf(details, "%s, n=%d, err=%d, w=%d", testname.c_str(), cnt, err, w);
@@ -1186,7 +1208,7 @@ void udp_server_sustained_4(UDP_UNIT_TEST_SIG_4)
 {
 	std::string testname = "udp_server_sustained_4";
 	std::string msg = "udp_sustained_4";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());	
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv4 address, RX (n/x)-datagrams\n");
 	int r, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -1218,8 +1240,7 @@ void udp_server_sustained_4(UDP_UNIT_TEST_SIG_4)
 		DEBUG_TEST("received DGRAM from %s : %d", inet_ntoa(in4->sin_addr), ntohs(in4->sin_port));
 		DEBUG_TEST("sending DGRAM(s) to %s : %d", inet_ntoa(remote_addr->sin_addr), ntohs(remote_addr->sin_port));
 	}
-	
+	sleep(ARTIFICIAL_SOCKET_LINGER);
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
 	//err = CLOSE(fd);
 	DEBUG_TEST("%s, n=%d, err=%d, r=%d", testname.c_str(), cnt, err, r);
 	sprintf(details, "%s, n=%d, err=%d, r=%d", testname.c_str(), cnt, err, r);
@@ -1235,7 +1256,7 @@ void udp_client_sustained_6(UDP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "udp_client_sustained_6";
 	std::string msg = "udp_sustained_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv6 address, TX n-datagrams\n");
 	int w, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -1267,7 +1288,7 @@ void udp_client_sustained_6(UDP_UNIT_TEST_SIG_6)
 			DEBUG_ERROR("error sending packet, err=%d", errno);
 		}
 	}
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	DEBUG_TEST("%s, n=%d, err=%d, w=%d", testname.c_str(), cnt, err, w);
 	sprintf(details, "%s, n=%d, err=%d, w=%d", testname.c_str(), cnt, err, w);
@@ -1284,7 +1305,7 @@ void udp_server_sustained_6(UDP_UNIT_TEST_SIG_6)
 {
 	std::string testname = "udp_server_sustained_6";
 	std::string msg = "udp_sustained_6";
-	fprintf(stderr, "\n\n%s\n", testname.c_str());	
+	fprintf(stderr, "\n\n%s (ts=%lu)\n", testname.c_str(), get_now_ts);
 	fprintf(stderr, "bind to interface with IPv6 address, RX (n/x)-datagrams\n");
 	int r, fd, err, len = strlen(msg.c_str());
 	char rbuf[STR_SIZE];
@@ -1316,8 +1337,8 @@ void udp_server_sustained_6(UDP_UNIT_TEST_SIG_6)
 		//DEBUG_TEST("received DGRAM from %s : %d", inet_ntoa(in6->sin6_addr), ntohs(in6->sin6_port));
 		//DEBUG_TEST("sending DGRAM(s) to %s : %d", inet_ntoa(remote_addr->sin6_addr), ntohs(remote_addr->sin6_port));
 	}
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
-	//err = CLOSE(fd);
+	err = CLOSE(fd);
 	DEBUG_TEST("%s, n=%d, err=%d, r=%d", testname.c_str(), cnt, err, r);
 	sprintf(details, "%s, n=%d, err=%d, r=%d", testname.c_str(), cnt, err, r);
 	DEBUG_TEST("Received : %s", rbuf);
@@ -1485,7 +1506,7 @@ void tcp_perf_tx_echo_4(TCP_UNIT_TEST_SIG_4)
 	float rate = (float)tot / (float)ts_delta;
 	sprintf(details, "%s, tot=%d, dt=%.2f, rate=%.2f MB/s", msg.c_str(), tot, ts_delta, (rate / float(ONE_MEGABYTE) ));
 
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	*passed = (tot == cnt && !err) ? PASSED : FAILED;
 }
@@ -1553,7 +1574,7 @@ void tcp_perf_rx_echo_4(TCP_UNIT_TEST_SIG_4)
 	float rate = (float)tot / (float)ts_delta;
 	sprintf(details, "%s, tot=%d, dt=%.2f, rate=%.2f MB/s", msg.c_str(), tot, ts_delta, (rate / float(ONE_MEGABYTE) ));		
 	
-	sleep(WAIT_FOR_TRANSMISSION_TO_COMPLETE);
+	sleep(ARTIFICIAL_SOCKET_LINGER);
 	err = CLOSE(fd);
 	*passed = (tot == cnt && !err) ? PASSED : FAILED;
 }
@@ -1567,6 +1588,7 @@ void tcp_perf_rx_echo_4(TCP_UNIT_TEST_SIG_4)
 
 int obscure_api_test(bool *passed)
 {
+	int err = -1;
 	fprintf(stderr, "\n\nobscure API test\n\n");	
 
 	/*
@@ -1610,52 +1632,71 @@ int obscure_api_test(bool *passed)
 
 	// TODO: write an ipv6 version of the above ^^^
 	*/
+/*
+int levels[] = {
+		IPPROTO_TCP, 
+		IPPROTO_UDP,
+		IPPROTO_IP
+	};
+	int num_levels = sizeof(levels) / sizeof(int);
 
-	// ---
+	int optnames[] = {
-	// Disable Nagle's Algorithm on a socket (TCP_NODELAY)
+		TCP_NODELAY,
-	int level = IPPROTO_TCP;
+		SO_LINGER
-	int optname = TCP_NODELAY;
+	};
-	int optval = 1;
+	int num_optnames = sizeof(optnames) / sizeof(int);
-	socklen_t flag_len = sizeof(optval);
+
-	int fd = SOCKET(AF_INET, SOCK_STREAM, 0);
+
-	DEBUG_TEST("setting level=%d, optname=%d, optval=%d...", level, optname, optval);
+	for (int i=0; i<num_levels; i++) { // test all levels
-	int err = SETSOCKOPT(fd, level, optname, (char *)&optval, sizeof(int));
+		for (int j=0; j<num_optnames; j++) { // test all optnames
-	if (err < 0) {
+
-		DEBUG_ERROR("error while setting optval on socket");
+		// ---
-		*passed = false;
+		// Disable Nagle's Algorithm on a socket (TCP_NODELAY)
-		err = -1;
+		int level = IPPROTO_TCP;
-	}
+		int optname = TCP_NODELAY;
-	optval = -99; // set junk value to test against
+		int optval = 1;
-	if ((err = GETSOCKOPT(fd, level, optname, &optval, &flag_len)) < 0) {
+		socklen_t flag_len = sizeof(optval);
-		DEBUG_ERROR("error while getting the optval");
+		int fd = SOCKET(AF_INET, SOCK_STREAM, 0);
-		*passed = false;
+		DEBUG_TEST("setting level=%d, optname=%d, optval=%d...", level, optname, optval);
-		err = -1;
+		err = SETSOCKOPT(fd, level, optname, (char *)&optval, sizeof(int));
-	}
+		if (err < 0) {
-	DEBUG_TEST("flag_len=%d", flag_len);
+			DEBUG_ERROR("error while setting optval on socket");
-	if (optval <= 0) {
-		DEBUG_ERROR("incorrect optval=%d (from getsockopt)", optval);
-		*passed = false;
-		err = -1;
-	} else {
-		DEBUG_TEST("correctly read optval=%d, now reversing it", optval);
-		if (optval > 0) { // TODO: what should be expected for each platform? Should this mirror them?
-			optval = 0;
-			DEBUG_TEST("setting level=%d, optname=%d, optval=%d...", level, optname, optval);
-			if ((err = SETSOCKOPT(fd, level, optname, (char *) &optval, (socklen_t)sizeof(int))) < 0) {
-				DEBUG_ERROR("error while setting on socket");
-				*passed = false;
-				err = -1;
-			}
-			else {
-				DEBUG_TEST("success");
-				*passed = true;
-			}
-		} else {
-			DEBUG_ERROR("the optval wasn't set correctly");
 			*passed = false;
 			err = -1;
 		}
+		optval = -99; // set junk value to test against
+		if ((err = GETSOCKOPT(fd, level, optname, &optval, &flag_len)) < 0) {
+			DEBUG_ERROR("error while getting the optval");
+			*passed = false;
+			err = -1;
+		}
+		DEBUG_TEST("flag_len=%d", flag_len);
+		if (optval <= 0) {
+			DEBUG_ERROR("incorrect optval=%d (from getsockopt)", optval);
+			*passed = false;
+			err = -1;
+		} else {
+			DEBUG_TEST("correctly read optval=%d, now reversing it", optval);
+			if (optval > 0) { // TODO: what should be expected for each platform? Should this mirror them?
+				optval = 0;
+				DEBUG_TEST("setting level=%d, optname=%d, optval=%d...", level, optname, optval);
+				if ((err = SETSOCKOPT(fd, level, optname, (char *) &optval, (socklen_t)sizeof(int))) < 0) {
+					DEBUG_ERROR("error while setting on socket");
+					*passed = false;
+					err = -1;
+				}
+				else {
+					DEBUG_TEST("success");
+					*passed = true;
+				}
+			} else {
+				DEBUG_ERROR("the optval wasn't set correctly");
+				*passed = false;
+				err = -1;
+			}
+		}
 	}
+	*/
 	return err;
 }
 
@@ -2468,18 +2509,23 @@ for (int i=0; i<num_repeats; i++)
 			//test_bad_args();
 
 	// OBSCURE API TEST
-	//obscure_api_test(&passed);
+	if (false) {
+		obscure_api_test(&passed);
+	}
 
-	// 
+	// Spam a SOCK_DGRAM socket from many threads
-	ipv = 4;
+	if (false) {
-	port = start_port;
+		ipv = 4;
-	str2addr(local_ipstr, port, ipv, (struct sockaddr *)&local_addr);
+		port = start_port;
-	str2addr(remote_ipstr, port, ipv, (struct sockaddr *)&remote_addr);
+		str2addr(local_ipstr, port, ipv, (struct sockaddr *)&local_addr);
-	multithread_udp_write((struct sockaddr_in *)&local_addr, (struct sockaddr_in *)&remote_addr, &passed);
+		str2addr(remote_ipstr, port, ipv, (struct sockaddr *)&remote_addr);
+		multithread_udp_write((struct sockaddr_in *)&local_addr, (struct sockaddr_in *)&remote_addr, &passed);
+	}
 	
 	//
-	multithread_test(10, &passed);
+	if (false) {
-	//exit(0);
+		multithread_test(10, &passed);
+	}
 
 #endif // __SELFTEST__
 
@@ -2506,7 +2552,7 @@ for (int i=0; i<num_repeats; i++)
 
 	#if defined(LIBZT_IPV4)
 		// UDP 4 client/server
-
+		
 		ipv = 4;
 		subtest_start_time_offset += subtest_expected_duration;
 		subtest_expected_duration = 30;