fixed comatose connection issue (was connection reuse at inappropriate times)

src/stack_drivers/lwip/lwip.cpp

@@ -1128,6 +1128,6 @@ namespace ZeroTier
                 break;
         }
         DEBUG_ERROR(" closing connection");
-        l->tap->closeConnection(l->conn);
+        l->tap->closeConnection(l->conn->sock);
     }
 }

src/stack_drivers/picotcp/picotcp.cpp

@@ -126,8 +126,8 @@ namespace ZeroTier {
 		DEBUG_INFO();
 		while(tap->_run)
 		{
-			tap->_phy.poll(25); // in ms
+			tap->_phy.poll(50); // in ms
-			usleep(50);
+			//usleep(50);
 	        tap->picostack->__pico_stack_tick();
 		}
 	}
@@ -144,7 +144,7 @@ namespace ZeroTier {
 	void pico_cb_tcp_read(NetconEthernetTap *tap, struct pico_socket *s)
 	{
 		// TODO: Verify 
-		DEBUG_INFO();
+		// DEBUG_INFO("picosock=%p", s);
 		Connection *conn = tap->getConnection(s);
 		if(conn) {
 			int r;				
@@ -161,7 +161,7 @@ namespace ZeroTier {
 		            r = tap->picostack->__pico_socket_recvfrom(s, conn->rxbuf + (conn->rxsz), ZT_MAX_MTU, (void *)&peer.ip4.addr, &port);
 		            // DEBUG_ATTN("received packet (%d byte) from %08X:%u", r, long_be2(peer.ip4.addr), short_be(port));
 		            tap->_phy.setNotifyWritable(conn->sock, true);
-		            DEBUG_EXTRA("read=%d", r);
+		            //DEBUG_EXTRA("read=%d", r);
 		            if (r > 0)
 		                conn->rxsz += r;
 	        	}
@@ -233,8 +233,8 @@ namespace ZeroTier {
 			if(r) {
 	            conn->rxsz += ZT_MAX_MTU;
 	            memcpy(sz_pos, &r, sizeof(r));
-	            tap->phyOnUnixWritable(conn->sock, NULL, false);
+	            tap->phyOnUnixWritable(conn->sock, NULL, true);
-	            tap->_phy.setNotifyWritable(conn->sock, false);
+	            //tap->_phy.setNotifyWritable(conn->sock, false);
 	        }
         	if (r < 0) {
             	DEBUG_ERROR("unable to read from picosock=%p", s);
@@ -256,9 +256,9 @@ namespace ZeroTier {
 
 		// Only called from a locked context, no need to lock anything
 		if(conn->txsz > 0) {
-			int r = conn->txsz < ZT_MAX_MTU ? conn->txsz : ZT_MAX_MTU;
+			int r, max_write_len = conn->txsz < ZT_MAX_MTU ? conn->txsz : ZT_MAX_MTU;
-			if((r = tap->picostack->__pico_socket_write(s, &conn->txbuf, r)) < 0) {
+			if((r = tap->picostack->__pico_socket_write(s, &conn->txbuf, max_write_len)) < 0) {
-				DEBUG_ERROR("unable to write to picosock=%p", (void*)s);
+				DEBUG_ERROR("unable to write to picosock=%p", s);
 				return;
 			}
 			int sz = (conn->txsz)-r;
@@ -283,12 +283,12 @@ namespace ZeroTier {
         }
         // Accept connection (analogous to lwip_nc_accept)
         if (ev & PICO_SOCK_EV_CONN) {
-            DEBUG_INFO("connection established with server, picosock=%p", (void*)(conn->picosock));
+            DEBUG_INFO("connection established with server, picosock=%p",(conn->picosock));
             uint32_t peer;
 			uint16_t port;
             struct pico_socket *client = picotap->picostack->__pico_socket_accept(s, &peer, &port);
             if(!client) {
-				DEBUG_EXTRA("unable to accept conn. (event might not be incoming, not necessarily an error), picosock=%p", (void*)(conn->picosock));
+				DEBUG_EXTRA("unable to accept conn. (event might not be incoming, not necessarily an error), picosock=%p", (conn->picosock));
 			}
 			ZT_PHY_SOCKFD_TYPE fds[2];
 			if(socketpair(PF_LOCAL, SOCK_STREAM, 0, fds) < 0) {
@@ -308,9 +308,10 @@ namespace ZeroTier {
 			if(sock_fd_write(fd, fds[1]) < 0) {
 				DEBUG_ERROR("error sending new fd to client application");
 			}
+			DEBUG_EXTRA("conn=%p, physock=%p, listen_picosock=%p, new_picosock=%p, fd=%d", newTcpConn, newTcpConn->sock, s, client, fds[1]);
         }
         if (ev & PICO_SOCK_EV_FIN) {
-            DEBUG_INFO("socket closed. exit normally.");
+            DEBUG_INFO("socket closed. exit normally. picosock=%p\n\n", s);
             //picotap->__pico_timer_add(2000, compare_results, NULL);
         }
         if (ev & PICO_SOCK_EV_ERR) {
@@ -318,8 +319,10 @@ namespace ZeroTier {
         }
         if (ev & PICO_SOCK_EV_CLOSE) {
             err = picotap->picostack->__pico_socket_close(s);
-            DEBUG_INFO("socket closure = %d", err);
+            DEBUG_INFO("socket closure = %d, picosock=%p", err, s);
-            picotap->closeConnection(conn);
+            if(err==0) {
+            	picotap->closeConnection(conn->sock);
+            }
             return;
         }
 
@@ -410,7 +413,7 @@ namespace ZeroTier {
 		memcpy(tap->pico_frame_rxbuf + tap->pico_frame_rxbuf_tot + sizeof(newlen) + sizeof(ethhdr), data, len);  // frame data
 		tap->pico_frame_rxbuf_tot += len + sizeof(len) + sizeof(ethhdr);
 		// DEBUG_INFO("RX frame buffer %3f full", (float)pico_frame_rxbuf_tot / (float)(1024 * 1024));
-		DEBUG_INFO("len=%d", len);
+		// DEBUG_INFO("len=%d", len);
 	}
 
 	// Called periodically by the stack, this removes data from the locked memory buffer and feeds it into the stack.
@@ -466,7 +469,7 @@ namespace ZeroTier {
 		}
 
 		if(psock) {
-			DEBUG_ATTN("physock=%p, picosock=%p", sock, (void*)psock);
+			DEBUG_ATTN("physock=%p, picosock=%p", sock, psock);
 			Connection * newConn = new Connection();
 	        *uptr = newConn;
 	        newConn->type = socket_rpc->socket_type;
@@ -500,7 +503,7 @@ namespace ZeroTier {
 
 		int max, r, max_write_len = conn->txsz < ZT_MAX_MTU ? conn->txsz : ZT_MAX_MTU;
 	    if((r = picotap->picostack->__pico_socket_write(conn->picosock, &conn->txbuf, max_write_len)) < 0) {
-	    	DEBUG_ERROR("unable to write to pico_socket(%p), r=%d", (conn->picosock), r);
+	    	DEBUG_ERROR("unable to write to picosock=%p, r=%d", (conn->picosock), r);
 	    	return;
 	    }
 	   
@@ -541,6 +544,7 @@ namespace ZeroTier {
 	    	DEBUG_TRANS("[UDP TX] --->    :: {TX: %.3f%%, RX: %.3f%%, physock=%p} :: %d bytes",
 	    		(float)conn->txsz / (float)max, (float)conn->rxsz / max, conn->sock, r);
 	    }
+	    check_buffer_states(conn);
     }
 
     // Instructs the stack to connect to a remote host
@@ -614,7 +618,7 @@ namespace ZeroTier {
 			ret = picotap->picostack->__pico_socket_bind(conn->picosock, &zaddr, (uint16_t*)&(addr->sin_port));
 		#endif
 		if(ret < 0) {
-			DEBUG_ERROR("unable to bind pico_socket(%p)", (void*)(conn->picosock));
+			DEBUG_ERROR("unable to bind pico_socket(%p), err=%d", (conn->picosock), ret);
 			if(ret == PICO_ERR_EINVAL) {
 				DEBUG_ERROR("PICO_ERR_EINVAL - invalid argument");
 				picotap->sendReturnValue(picotap->_phy.getDescriptor(rpcSock), -1, EINVAL);
@@ -635,12 +639,12 @@ namespace ZeroTier {
     void pico_handleListen(PhySocket *sock, PhySocket *rpcSock, void **uptr, struct listen_st *listen_rpc)
     {
     	Connection *conn = picotap->getConnection(sock);
-    	DEBUG_ATTN("conn = %p", (void*)conn);
+    	DEBUG_ATTN("physock=%p, conn=%p, picosock=%p", sock, conn, conn->picosock);
     	if(!sock || !conn) {
     		DEBUG_ERROR("invalid connection");
     		return;
     	}
-    	int ret, backlog = 1;
+    	int ret, backlog = 100;
     	if((ret = picotap->picostack->__pico_socket_listen(conn->picosock, backlog)) < 0)
     	{
     		if(ret == PICO_ERR_EINVAL) {
@@ -664,12 +668,10 @@ namespace ZeroTier {
     // ----------------------------------------- 
     void pico_handleRead(PhySocket *sock,void **uptr,bool lwip_invoked)
     {
-    	/*
         if(!lwip_invoked) {
             picotap->_tcpconns_m.lock();
-            //picotap->_rx_buf_m.lock(); 
+            picotap->_rx_buf_m.lock(); 
         }
-		*/
 		
     	DEBUG_ATTN();
 		Connection *conn = picotap->getConnection(sock);
@@ -680,7 +682,7 @@ namespace ZeroTier {
 			
 			if(conn->type==SOCK_DGRAM) {
 				n = picotap->_phy.streamSend(conn->sock, conn->rxbuf, ZT_MAX_MTU);
-				DEBUG_EXTRA("SOCK_DGRAM, physock=%p", sock);
+				DEBUG_EXTRA("SOCK_DGRAM, conn=%p, physock=%p", conn, sock);
 				int payload_sz, addr_sz_offset = sizeof(struct sockaddr_storage);
 				memcpy(&payload_sz, conn->rxbuf + addr_sz_offset, sizeof(int));
 				struct sockaddr_storage addr;
@@ -693,7 +695,7 @@ namespace ZeroTier {
 			
 			if(conn->type==SOCK_STREAM) {
 				n = picotap->_phy.streamSend(conn->sock, conn->rxbuf, conn->rxsz);	
-				DEBUG_EXTRA("SOCK_STREAM, physock=%p", sock);
+				DEBUG_EXTRA("SOCK_STREAM, conn=%p, physock=%p, n=%d", conn, sock, n);
 				if(conn->rxsz-n > 0) // If more remains on buffer
 					memcpy(conn->rxbuf, conn->rxbuf+n, conn->rxsz - n);
 			  	conn->rxsz -= n;
@@ -701,22 +703,33 @@ namespace ZeroTier {
 			if(n) {
 				if(conn->type==SOCK_STREAM) {
 	            	DEBUG_TRANS("[TCP RX] <---    :: {TX: %.3f%%, RX: %.3f%%, physock=%p} :: %ld bytes",
-	                	(float)conn->txsz / max, (float)conn->rxsz / max, (void*)conn->sock, n);
+	                	(float)conn->txsz / max, (float)conn->rxsz / max, conn->sock, n);
-	        	}
-	        	picotap->_phy.setNotifyWritable(conn->sock, true);
 	        	}
 	        	if(conn->rxsz == 0) {
-				picotap->_phy.setNotifyWritable(conn->sock, false);
+					picotap->_phy.setNotifyWritable(sock, false);
+				}
+				else {
+	        		picotap->_phy.setNotifyWritable(sock, true);	
 				}
 			}
+			else {
+				picotap->_phy.setNotifyWritable(sock, false);
+			}	
+		}
+		picotap->_phy.whack();
 		check_buffer_states(conn);
+
+		if(!lwip_invoked) {
+            picotap->_tcpconns_m.unlock();
+            picotap->_rx_buf_m.unlock();
+        }
     }
 
     // Closes a pico_socket
-    /*
+    void pico_handleClose(PhySocket *sock)
-    static void pico_handleClose(Connection *conn)
     {
     	DEBUG_INFO();
+    	/*
     	int ret;
     	if(conn && conn->picosock) {
 	    	if((ret = picotap->picostack->__pico_socket_close(conn->picosock)) < 0) {
@@ -726,9 +739,8 @@ namespace ZeroTier {
 	    	return;
     	}
     	DEBUG_ERROR("invalid connection or pico_socket");
-    }
     	*/
-
+    }
 }
 
 #endif // SDK_PICOTCP

src/stack_drivers/picotcp/picotcp.hpp

@@ -111,6 +111,8 @@ namespace ZeroTier {
     void pico_handleBind(PhySocket *sock, PhySocket *rpcSock, void **uptr, struct bind_st *bind_rpc);
     void pico_handleListen(PhySocket *sock, PhySocket *rpcSock, void **uptr, struct listen_st *listen_rpc);
     void pico_handleRead(PhySocket *sock,void **uptr,bool lwip_invoked);
+    void pico_handleClose(PhySocket *sock);
+
 
     /**
      * Loads an instance of picoTCP stack library in a private memory arena
@@ -277,19 +279,19 @@ namespace ZeroTier {
         inline int __pico_string_to_ipv6(PICO_STRING_TO_IPV6_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_string_to_ipv6(ipstr, ip); }
         inline int __pico_socket_setoption(PICO_SOCKET_SETOPTION_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_setoption(s, option, value); }
         inline uint32_t __pico_timer_add(PICO_TIMER_ADD_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_timer_add(expire, timer, arg); }
-        inline int __pico_socket_send(PICO_SOCKET_SEND_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_send(s, buf, len); }
+        inline int __pico_socket_send(PICO_SOCKET_SEND_SIG) throw() { /*DEBUG_STACK();*/ Mutex::Lock _l(_lock); return _pico_socket_send(s, buf, len); }
-        inline int __pico_socket_sendto(PICO_SOCKET_SENDTO_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_sendto(s, buf, len, dst, remote_port); }
+        inline int __pico_socket_sendto(PICO_SOCKET_SENDTO_SIG) throw() { /*DEBUG_STACK();*/ Mutex::Lock _l(_lock); return _pico_socket_sendto(s, buf, len, dst, remote_port); }
-        inline int __pico_socket_recv(PICO_SOCKET_RECV_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_recv(s, buf, len); }
+        inline int __pico_socket_recv(PICO_SOCKET_RECV_SIG) throw() { /*DEBUG_STACK();*/ Mutex::Lock _l(_lock); return _pico_socket_recv(s, buf, len); }
-        inline int __pico_socket_recvfrom(PICO_SOCKET_RECVFROM_SIG) throw() { DEBUG_STACK(); /*Mutex::Lock _l(_lock);*/ return _pico_socket_recvfrom(s, buf, len, orig, remote_port); }
+        inline int __pico_socket_recvfrom(PICO_SOCKET_RECVFROM_SIG) throw() { /*DEBUG_STACK();*/ /*Mutex::Lock _l(_lock);*/ return _pico_socket_recvfrom(s, buf, len, orig, remote_port); }
-        inline struct pico_socket * __pico_socket_open(PICO_SOCKET_OPEN_SIG) throw() { DEBUG_STACK();  return _pico_socket_open(net, proto, wakeup); }
+        inline struct pico_socket * __pico_socket_open(PICO_SOCKET_OPEN_SIG) throw() { DEBUG_ATTN();  return _pico_socket_open(net, proto, wakeup); }
-        inline int __pico_socket_bind(PICO_SOCKET_BIND_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_bind(s, local_addr, port); }
+        inline int __pico_socket_bind(PICO_SOCKET_BIND_SIG) throw() { DEBUG_ATTN(); Mutex::Lock _l(_lock); return _pico_socket_bind(s, local_addr, port); }
-        inline int __pico_socket_connect(PICO_SOCKET_CONNECT_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_connect(s, srv_addr, remote_port); }
+        inline int __pico_socket_connect(PICO_SOCKET_CONNECT_SIG) throw() { DEBUG_ATTN(); Mutex::Lock _l(_lock); return _pico_socket_connect(s, srv_addr, remote_port); }
-        inline int __pico_socket_listen(PICO_SOCKET_LISTEN_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_listen(s, backlog); }
+        inline int __pico_socket_listen(PICO_SOCKET_LISTEN_SIG) throw() { DEBUG_ATTN(); Mutex::Lock _l(_lock); return _pico_socket_listen(s, backlog); }
-        inline int __pico_socket_read(PICO_SOCKET_READ_SIG) throw() { DEBUG_STACK(); /*Mutex::Lock _l(_lock); */ return _pico_socket_read(s, buf, len); }
+        inline int __pico_socket_read(PICO_SOCKET_READ_SIG) throw() { /*DEBUG_STACK();*/ /*Mutex::Lock _l(_lock); */ return _pico_socket_read(s, buf, len); }
-        inline int __pico_socket_write(PICO_SOCKET_WRITE_SIG) throw() { DEBUG_STACK(); /*Mutex::Lock _l(_lock);*/ return _pico_socket_write(s, buf, len); }
+        inline int __pico_socket_write(PICO_SOCKET_WRITE_SIG) throw() { /*DEBUG_STACK();*/ /*Mutex::Lock _l(_lock);*/ return _pico_socket_write(s, buf, len); }
         inline int __pico_socket_close(PICO_SOCKET_CLOSE_SIG) throw() { DEBUG_STACK(); /*Mutex::Lock _l(_lock);*/ return _pico_socket_close(s); }
         inline int __pico_socket_shutdown(PICO_SOCKET_SHUTDOWN_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_socket_shutdown(s, mode); }
-        inline struct pico_socket * __pico_socket_accept(PICO_SOCKET_ACCEPT_SIG) throw() { DEBUG_STACK(); /*Mutex::Lock _l(_lock);*/ return _pico_socket_accept(s, orig, port); }
+        inline struct pico_socket * __pico_socket_accept(PICO_SOCKET_ACCEPT_SIG) throw() { DEBUG_ATTN(); /*Mutex::Lock _l(_lock);*/ return _pico_socket_accept(s, orig, port); }
         inline int __pico_ipv6_link_add(PICO_IPV6_LINK_ADD_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock); return _pico_ipv6_link_add(dev, address, netmask); }
         //inline struct pico_ipv6 * __pico_ipv6_source_find(PICO_IPV6_SOURCE_FIND_SIG) throw() { DEBUG_STACK(); Mutex::Lock _l(_lock)l return _pico_ipv6_source_find(dst); }
     };

src/tap.cpp

@@ -296,7 +296,7 @@ Connection *NetconEthernetTap::getConnection(struct pico_socket *sock)
 
 void NetconEthernetTap::closeConnection(PhySocket *sock)
 {
-    DEBUG_EXTRA("physock=%p", (void*)sock);
+    DEBUG_EXTRA("physock=%p", sock);
 	Mutex::Lock _l(_close_m);
 	// Here we assume _tcpconns_m is already locked by caller
 	if(!sock) {
@@ -305,7 +305,7 @@ void NetconEthernetTap::closeConnection(PhySocket *sock)
 	}
 	// picoTCP
 	#if defined(SDK_PICOTCP)
-		// pico_handleClose(conn);
+		pico_handleClose(sock);
 	#endif
     Connection *conn = getConnection(sock);
     if(!conn)
@@ -328,7 +328,7 @@ void NetconEthernetTap::closeConnection(PhySocket *sock)
 }
 
 void NetconEthernetTap::phyOnUnixClose(PhySocket *sock,void **uptr) {
-    DEBUG_EXTRA("physock=%p", (void*)&sock);
+    DEBUG_EXTRA("physock=%p", sock);
 	Mutex::Lock _l(_tcpconns_m);
     //closeConnection(sock);
 }
@@ -343,7 +343,7 @@ void NetconEthernetTap::phyOnUnixClose(PhySocket *sock,void **uptr) {
 // ----------------------------------------- 
 void NetconEthernetTap::handleRead(PhySocket *sock,void **uptr,bool lwip_invoked)
 {
-	// DEBUG_EXTRA("handleRead(physock=%p): lwip_invoked = %d\n", (void*)&sock, lwip_invoked);
+	// DEBUG_EXTRA("handleRead(physock=%p): lwip_invoked = %d\n", sock, lwip_invoked);
 	#if defined(SDK_PICOTCP)
 		pico_handleRead(sock, uptr, lwip_invoked);
 	#endif
@@ -359,7 +359,7 @@ void NetconEthernetTap::phyOnUnixWritable(PhySocket *sock,void **uptr,bool lwip_
 
 void NetconEthernetTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data, ssize_t len)
 {
-    DEBUG_EXTRA("physock=%p, len=%d", (void*)&sock, (int)len);
+    DEBUG_EXTRA("physock=%p, len=%d", sock, (int)len);
 	uint64_t CANARY_num;
 	pid_t pid, tid;
 	ssize_t wlen = len;
@@ -381,10 +381,10 @@ void NetconEthernetTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data,
 	if(detected_rpc) {
 		unloadRPC(data, pid, tid, timestamp, CANARY, cmd, payload);
 		memcpy(&CANARY_num, CANARY, CANARY_SZ);
-		// DEBUG_EXTRA(" RPC: physock=%p, (pid=%d, tid=%d, timestamp=%s, cmd=%d)", (void*)&sock, pid, tid, timestamp, cmd);
+		// DEBUG_EXTRA(" RPC: physock=%p, (pid=%d, tid=%d, timestamp=%s, cmd=%d)", sock, pid, tid, timestamp, cmd);
 
 		if(cmd == RPC_SOCKET) {				
-			DEBUG_INFO("RPC_SOCKET, physock=%p", (void*)&sock);
+			DEBUG_INFO("RPC_SOCKET, physock=%p", sock);
 			// Create new lwip socket and associate it with this sock
 			struct socket_st socket_rpc;
 			memcpy(&socket_rpc, &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct socket_st));
@@ -453,6 +453,7 @@ void NetconEthernetTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data,
 		}
 		// Write data from stream
         if(wlen) {
+        	/*
             if(conn->type == SOCK_STREAM) { // We only disable TCP "connections"
                 int softmax = conn->type == SOCK_STREAM ? DEFAULT_TCP_RX_BUF_SZ : DEFAULT_UDP_RX_BUF_SZ;
                 if(conn->txsz > softmax) {
@@ -464,6 +465,7 @@ void NetconEthernetTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data,
                     _phy.setNotifyReadable(sock, true);
                 }
             }
+            */
             conn->txsz += wlen;
             handleWrite(conn);
         }
@@ -473,34 +475,34 @@ void NetconEthernetTap::phyOnUnixData(PhySocket *sock, void **uptr, void *data,
         rpcSock = sockdata.first;
         buf = (unsigned char*)sockdata.second;
 		unloadRPC(buf, pid, tid, timestamp, CANARY, cmd, payload);
-		// DEBUG_EXTRA(" RPC: physock=%p, (pid=%d, tid=%d, timestamp=%s, cmd=%d)", (void*)&sock, pid, tid, timestamp, cmd);
+		// DEBUG_EXTRA(" RPC: physock=%p, (pid=%d, tid=%d, timestamp=%s, cmd=%d)", sock, pid, tid, timestamp, cmd);
 		switch(cmd) {
 			case RPC_BIND:
-				DEBUG_INFO("RPC_BIND, physock=%p", (void*)&sock);
+				DEBUG_INFO("RPC_BIND, physock=%p", sock);
 			    struct bind_st bind_rpc;
 			    memcpy(&bind_rpc,  &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct bind_st));
 			    handleBind(sock, rpcSock, uptr, &bind_rpc);
 				break;
 		  	case RPC_LISTEN:
-		  		DEBUG_INFO("RPC_LISTEN, physock=%p", (void*)&sock);
+		  		DEBUG_INFO("RPC_LISTEN, physock=%p", sock);
 			    struct listen_st listen_rpc;
 			    memcpy(&listen_rpc,  &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct listen_st));
 			    handleListen(sock, rpcSock, uptr, &listen_rpc);
 				break;
 		  	case RPC_GETSOCKNAME:
-		  		DEBUG_INFO("RPC_GETSOCKNAME, physock=%p", (void*)&sock);
+		  		DEBUG_INFO("RPC_GETSOCKNAME, physock=%p", sock);
 		  		struct getsockname_st getsockname_rpc;
 		    	memcpy(&getsockname_rpc,  &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct getsockname_st));
 		  		handleGetsockname(sock, rpcSock, uptr, &getsockname_rpc);
 		  		break;
 			case RPC_GETPEERNAME:
-		  		DEBUG_INFO("RPC_GETPEERNAME, physock=%p", (void*)&sock);
+		  		DEBUG_INFO("RPC_GETPEERNAME, physock=%p", sock);
 		  		struct getsockname_st getpeername_rpc;
 		    	memcpy(&getpeername_rpc,  &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct getsockname_st));
 		  		handleGetpeername(sock, rpcSock, uptr, &getpeername_rpc);
 		  		break;
 			case RPC_CONNECT:
-				DEBUG_INFO("RPC_CONNECT, physock=%p", (void*)&sock);
+				DEBUG_INFO("RPC_CONNECT, physock=%p", sock);
 			    struct connect_st connect_rpc;
 			    memcpy(&connect_rpc,  &buf[IDX_PAYLOAD+STRUCT_IDX], sizeof(struct connect_st));
 			    handleConnect(sock, rpcSock, conn, &connect_rpc);
@@ -572,7 +574,7 @@ int NetconEthernetTap::handleConnectProxy(PhySocket *sock, struct sockaddr_in *r
 // Connect a stack's PCB/socket/Connection object to a remote host
 void NetconEthernetTap::handleConnect(PhySocket *sock, PhySocket *rpcSock, Connection *conn, struct connect_st* connect_rpc)
 {
-    DEBUG_ATTN("physock=%p", (void*)&sock);
+    DEBUG_ATTN("physock=%p", sock);
 	Mutex::Lock _l(_tcpconns_m);
 	#if defined(SDK_PICOTCP)
 		pico_handleConnect(sock, rpcSock, conn, connect_rpc);		

tests/api_test/multiclient.c

@@ -0,0 +1,60 @@
+// TCP Client test program
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+ 
+int atoi(const char *str);
+int close(int filedes);
+
+#define MSG_SZ 128
+
+int main(int argc , char *argv[])
+{
+    if(argc < 3) {
+        printf("usage: client <addr> <port>\n");
+        return 1;
+    }
+   
+    int sock, port = atoi(argv[2]);
+    struct sockaddr_in server;
+    char message[MSG_SZ] , server_reply[MSG_SZ];
+     
+    sock = socket(AF_INET , SOCK_STREAM , 0);
+    if (sock == -1) {
+        printf("could not create socket");
+    }     
+    server.sin_addr.s_addr = inet_addr(argv[1]);
+    server.sin_family = AF_INET;
+    server.sin_port = htons( port );
+ 
+    printf("connecting...\n");
+    if (connect(sock , (struct sockaddr *)&server , sizeof(server)) < 0) {
+        perror("connect failed. Error");
+        return 1;
+    }
+    printf("\n\n\nconnected\n");
+
+    char *msg = "welcome to the machine!";
+    
+    // TX         
+    if(send(sock, msg, strlen(msg), 0) < 0) {
+        printf("send failed");
+        return 1;
+    }
+    else {
+        printf("TX: %s\n", msg);
+        printf("len = %ld\n", strlen(msg));
+
+        int bytes_read = read(sock, server_reply, MSG_SZ);
+        if(bytes_read < 0)
+            printf("\tRX: Nothing\n");
+        else
+            printf("\tRX = (%d bytes): %s\n", bytes_read, server_reply);
+    }
+    
+    close(sock);
+    return 0;
+}

tests/api_test/multiserve.c

@@ -0,0 +1,66 @@
+// TCP Server test program
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <arpa/inet.h> 
+#include <unistd.h>  
+ 
+int atoi(const char *str);
+
+int main(int argc , char *argv[])
+{
+    if(argc < 2) {
+       printf("usage: tcp_server <port>\n");
+       return 0;
+    }
+
+    int sock, client_sock, c, read_size, port = atoi(argv[1]);
+    char client_message[2000];
+     
+    char str[100];
+    int comm_fd;
+ 
+    struct sockaddr_in servaddr;
+        struct sockaddr_in client;
+
+    sock = socket(AF_INET, SOCK_STREAM, 0); 
+    bzero( &servaddr, sizeof(servaddr));
+ 
+    servaddr.sin_family = AF_INET;
+    servaddr.sin_addr.s_addr = htons(INADDR_ANY);
+    servaddr.sin_port = htons(port);
+    bind(sock, (struct sockaddr *) &servaddr, sizeof(servaddr));
+
+    printf("listening\n");
+    listen(sock , 3); 
+    printf("waiting to accept\n");
+    c = sizeof(struct sockaddr_in);
+    
+
+    while(1)
+    {
+      client_sock = accept(sock, (struct sockaddr *)&client, (socklen_t*)&c);
+      if (client_sock < 0) {
+          perror("accept failed");
+          return 0;
+      }
+      printf("\n\n\nconnection accepted\n reading...\n");
+     
+      // RX
+
+      int msglen = 1024;
+      unsigned long count = 0;
+
+     int bytes_read = read(client_sock, client_message, msglen);
+     printf("[%lu] RX = (%d): ", count, bytes_read);
+     for(int i=0; i<bytes_read; i++) {
+        printf("%c", client_message[i]);
+     }
+
+     // TX
+     int bytes_written = write(client_sock, "Server here!", 12); 
+     printf("\t\nTX = %d\n", bytes_written);  
+    }
+    return 0;
+}