Implemented barebones VL2 SOCK_RAW functionality, moved ZT core helper functions from OneService to libzt

include/libzt.h

@@ -37,9 +37,44 @@
 /* work on other platforms. Mayber later down the road we will customize    */
 /* this for each different platform. Maybe.                                 */
 /****************************************************************************/
+
 #if !defined(__linux__)
 #define SIOCGIFINDEX  101
 #define SIOCGIFHWADDR 102
+
+// Normally defined in linux/if_packet.h, defined here so we can offer a linux-like 
+// raw socket API on non-linux platforms 
+struct sockaddr_ll {
+    unsigned short sll_family;   /* Always AF_PACKET */
+    unsigned short sll_protocol; /* Physical layer protocol */
+    int            sll_ifindex;  /* Interface number */
+    unsigned short sll_hatype;   /* ARP hardware type */
+    unsigned char  sll_pkttype;  /* Packet type */
+    unsigned char  sll_halen;    /* Length of address */
+    unsigned char  sll_addr[8];  /* Physical layer address */
+};
+
+/*
+struct zts_ifreq {
+   char ifr_name[IFNAMSIZ]; // Interface name
+   union {
+       struct sockaddr ifr_addr;
+       struct sockaddr ifr_dstaddr;
+       struct sockaddr ifr_broadaddr;
+       struct sockaddr ifr_netmask;
+       struct sockaddr ifr_hwaddr;
+       short           ifr_flags;
+       int             ifr_ifindex;
+       int             ifr_metric;
+       int             ifr_mtu;
+       struct ifmap    ifr_map;
+       char            ifr_slave[IFNAMSIZ];
+       char            ifr_newname[IFNAMSIZ];
+       char           *ifr_data;
+   };
+};
+*/
+
 #endif
 
 /****************************************************************************/
@@ -437,6 +472,9 @@ namespace ZeroTier
 
   class lwIP;
   extern ZeroTier::lwIP *lwipstack;
+
+  class SocketTap;
+  struct InetAddress;
 }
 
 /*
@@ -461,6 +499,17 @@ int zts_nsockets();
 int zts_maxsockets();
 
 int pico_ntimers();
+
+/****************************************************************************/
+/* ZeroTier Core helper functions for libzt - DON'T CALL THESE DIRECTLY     */
+/****************************************************************************/
+
+ZeroTier::SocketTap *getTapByNWID(uint64_t nwid);
+ZeroTier::SocketTap *getTapByAddr(ZeroTier::InetAddress &addr);
+ZeroTier::SocketTap *getTapByName(char *ifname);
+ZeroTier::SocketTap *getTapByIndex(int index);
+void dismantleTaps();
+
 /**
  * Don't call this directly, use 'zts_start()'
  */

make-liblwip.mk

@@ -45,7 +45,7 @@ ifeq ($(origin CCX),default)
 	CCX=$(shell if [ -e /usr/bin/clang++ ]; then echo clang++; else echo g++; fi)
 endif
 
-CFLAGS=-O3 -g -Wall -DIPv4 -fPIC
+CFLAGS=-O3 -g -Wall -DIPv4 -DIPv6 -fPIC
 
 CFLAGS:=$(CFLAGS) \
 	-I$(LWIPDIR)/include -I$(LWIPARCH)/include -I$(LWIPDIR)/include/ipv4 \
@@ -115,11 +115,15 @@ ARCHFILES=$(wildcard $(LWIPARCH)/*.c $(LWIPARCH)tapif.c $(LWIPARCH)/netif/list.c
 # LWIPFILES: All the above.
 LWIPFILES=$(COREFILES) $(CORE4FILES) $(APIFILES) $(NETIFFILES) $(ARCHFILES)
 
-# ipv6 support
+ifeq ($(IPV4),1)
-ifeq ($(SDK_IPV6),1)
+	LWIPFILES+=$(CORE4FILES)
-	LWIPFILES+=$(CORE6FILES)
+	CFLAGS+=-DLIBZT_IPV4=1
-	CFLAGS+=-DSDK_IPV6=1
 endif
+ifeq ($(IPV6),1)
+	LWIPFILES+=$(CORE6FILES)
+	CFLAGS+=-DLIBZT_IPV6=1
+endif
+
 LWIPFILESW=$(wildcard $(LWIPFILES))
 LWIPOBJS=$(notdir $(LWIPFILESW:.c=.o))
 
@@ -145,9 +149,6 @@ $(LWIPLIB): $(LWIPOBJS)
 liblwip.a: $(LWIPOBJS)
 	echo $(LWIPOBJS)
 	libtool -static -o $@ $^
-	#echo "wtf"
-	#echo $(NETIFFILES)
-	#$(AR) -r $@ $^
 
 .depend: $(LWIPFILES)
 	$(CCDEP) $(CFLAGS) -MM $^ > .depend || rm -f .depend

make-linux.mk

@@ -201,7 +201,7 @@ picotcp:
 	cd $(STACK_DIR); make lib ARCH=shared IPV4=1 IPV6=1
 
 lwip:
-	-make -f make-liblwip.mk liblwip.a
+	-make -f make-liblwip.mk liblwip.a IPV4=1 IPV6=1
 
 ##############################################################################
 ## Static Libraries                                                         ##
@@ -257,8 +257,7 @@ tests: $(UNIT_TEST_OBJ_FILES)
 # Cleans only current $(OSTYPE)
 clean:
 	-rm -rf $(BUILD)/*
-	-find 'ext/picotcp' -type f \( -name '*.o' -o -name '*.so' -o -name '*.a' \) -delete
+	-find . -type f \( -name '*.a' -o -name '*.o' -o -name '*.so' -o -name '*.o.d' -o -name '*.out' -o -name '*.log' -o -name '*.dSYM' \) -delete
-	-find . -type f \( -name '*.o' -o -name '*.so' -o -name '*.o.d' -o -name '*.out' -o -name '*.log' -o -name '*.dSYM' \) -delete
 
 # Clean everything
 nuke:

make-mac.mk

@@ -269,8 +269,7 @@ tests: $(UNIT_TEST_OBJ_FILES)
 # Cleans only current $(OSTYPE)
 clean:
 	-rm -rf $(BUILD)/*
-	-find $(PICO_DIR) -type f \( -name '*.o' -o -name '*.so' -o -name '*.a' \) -delete
+	-find . -type f \( -name '*.a' -o -name '*.o' -o -name '*.so' -o -name '*.o.d' -o -name '*.out' -o -name '*.log' -o -name '*.dSYM' \) -delete
-	-find . -type f \( -name '*.o' -o -name '*.so' -o -name '*.o.d' -o -name '*.out' -o -name '*.log' -o -name '*.dSYM' \) -delete
 
 # Clean everything
 nuke:

objects.mk

@@ -1,11 +1,8 @@
 ZTO_OBJS=\
-	zto/controller/EmbeddedNetworkController.o \
-	zto/controller/JSONDB.o \
 	zto/node/C25519.o \
 	zto/node/Capability.o \
 	zto/node/CertificateOfMembership.o \
 	zto/node/CertificateOfOwnership.o \
-	zto/node/Cluster.o \
 	zto/node/Identity.o \
 	zto/node/IncomingPacket.o \
 	zto/node/InetAddress.o \
@@ -26,12 +23,13 @@ ZTO_OBJS=\
 	zto/node/Switch.o \
 	zto/node/Tag.o \
 	zto/node/Topology.o \
+	zto/node/Trace.o \
 	zto/node/Utils.o \
+	zto/controller/EmbeddedNetworkController.o \
+	zto/controller/JSONDB.o \
 	zto/osdep/ManagedRoute.o \
 	zto/osdep/Http.o \
 	zto/osdep/OSUtils.o \
-	zto/service/ClusterGeoIpService.o \
 	zto/service/SoftwareUpdater.o \
 	zto/service/OneService.o \
 	zto/ext/http-parser/http_parser.o
-

src/Connection.hpp

@@ -66,7 +66,7 @@ namespace ZeroTier {
 		struct sockaddr_storage *local_addr; // Address we've bound to locally
 		struct sockaddr_storage *peer_addr;  // Address of connection call to remote host
 
-		int socket_family, socket_type;
+		int socket_family, socket_type, protocol;
 
 		int app_fd; // used by app for I/O
 		int sdk_fd; // used by lib for I/O

src/SocketTap.cpp

@@ -24,6 +24,10 @@
  * of your own application.
  */
 
+#include <netinet/in.h>
+#include <net/if_arp.h>
+#include <arpa/inet.h>
+
 #include <algorithm>
 #include <utility>
 #include <sys/poll.h>
@@ -41,13 +45,26 @@
 #include "lwIP.hpp"
 #endif
 
+#if defined(__APPLE__)
+#include <net/ethernet.h>
+#endif
+#if defined(__linux__)
+#include <netinet/ether.h>
+#endif
+
 #include "Utils.hpp"
 #include "OSUtils.hpp"
 #include "Constants.hpp"
 #include "Phy.hpp"
 
+class SocketTap;
+
+extern std::vector<void*> vtaps;
+
 namespace ZeroTier {
 
+	int SocketTap::devno = 0;
+
 	/****************************************************************************/
 	/* SocketTap Service                                                        */
 	/* - For each joined network a SocketTap will be created to administer I/O  */
@@ -75,6 +92,15 @@ namespace ZeroTier {
 			_unixListenSocket((PhySocket *)0),
 			_phy(this,false,true)
 	{
+		vtaps.push_back((void*)this);
+
+		// set interface name
+		char tmp3[17];
+		ifindex = devno;
+		sprintf(tmp3, "libzt%d", devno++);
+		_dev = tmp3;
+		DEBUG_INFO("set device name to: %s", _dev.c_str());
+
 		_thread = Thread::start(this);
 	}
 
@@ -175,6 +201,7 @@ namespace ZeroTier {
 
 	void SocketTap::setFriendlyName(const char *friendlyName) 
 	{
+		DEBUG_INFO("%s", friendlyName);
 		// Someday
 	}
 
@@ -300,52 +327,57 @@ namespace ZeroTier {
 		return NULL;
 	}
 
-	void SocketTap::Read(PhySocket *sock,void **uptr,bool stack_invoked) {
+	int SocketTap::Read(PhySocket *sock,void **uptr,bool stack_invoked) {
 #if defined(STACK_PICO)
-		DEBUG_INFO();
 		if(picostack)
-			picostack->pico_Read(this, sock, (Connection*)uptr, stack_invoked);
+			return picostack->pico_Read(this, sock, (Connection*)uptr, stack_invoked);
 #endif
+		return -1;
 	}
 
-	void SocketTap::Write(Connection *conn, void *data, ssize_t len) {
+	int SocketTap::Write(Connection *conn, void *data, ssize_t len) {
+		if(conn->socket_type == SOCK_RAW) { // we don't want to use a stack, just VL2
+        	struct ether_header *eh = (struct ether_header *) data;
+        	MAC src_mac;
+        	MAC dest_mac;
+        	src_mac.setTo(eh->ether_shost, 6);
+        	dest_mac.setTo(eh->ether_dhost, 6);
+        	_handler(_arg,NULL,_nwid,src_mac,dest_mac, Utils::ntoh((uint16_t)eh->ether_type),0, ((char*)data) + sizeof(struct ether_header),len - sizeof(struct ether_header));
+        	return len;
+		}
+
 #if defined(STACK_PICO)
-		//DEBUG_INFO();
 		if(picostack)
-			picostack->pico_Write(conn, data, len);
+			return picostack->pico_Write(conn, data, len);
 #endif
+		return -1;
 	}
 
-	void SocketTap::Close(Connection *conn) {
+	int SocketTap::Close(Connection *conn) {
 #if defined(STACK_PICO)
 		if(!conn) {
 			DEBUG_ERROR("invalid connection");
-			return;
+			return -1;
 		}
-		//DEBUG_INFO("A");
 		picostack->pico_Close(conn);
 		if(!conn->sock) {
 			// DEBUG_EXTRA("invalid PhySocket");
-			return;
+			return -1;
 		}
 		// Here we assume _tcpconns_m is already locked by caller
 		// FIXME: is this assumption still valid
 		if(conn->state==ZT_SOCK_STATE_LISTENING)
 		{
-			//DEBUG_INFO("B");
 			// since we never wrapped this socket
 			DEBUG_INFO("in LISTENING state, no need to close in PhyIO");
-			return;
+			return -1;
 		}
 		else
 		{
-			//DEBUG_INFO("C");
 			if(conn->sock)
 				_phy.close(conn->sock, false);
 		}
 		close(_phy.getDescriptor(conn->sock));
-
-		//DEBUG_INFO("D");
 		for(size_t i=0;i<_Connections.size();++i) {
 			if(_Connections[i] == conn){
 				// FIXME: double free issue exists here (potentially)
@@ -355,6 +387,7 @@ namespace ZeroTier {
 			}
 		}
 #endif
+		return 0; // TODO
 	}
 
 	void SocketTap::Housekeeping()

src/SocketTap.hpp

@@ -183,6 +183,9 @@ namespace ZeroTier {
         netif lwipdev6;
 #endif
 
+		static int devno;
+		int ifindex;
+
 		std::vector<InetAddress> ips() const;
 		std::vector<InetAddress> _ips;
 
@@ -239,17 +242,17 @@ namespace ZeroTier {
 		/* 
 		 * Move data from RX buffer to application's "socket"
 		 */
-		void Read(PhySocket *sock,void **uptr,bool stack_invoked);
+		int Read(PhySocket *sock,void **uptr,bool stack_invoked);
 
 		/* 
 	 	 * Move data from application's "socket" into network stack
 	 	 */
-		void Write(Connection *conn, void *data, ssize_t len);
+		int Write(Connection *conn, void *data, ssize_t len);
 
 		/*
 		 * Closes a Connection
 		 */
-        void Close(Connection *conn);
+        int Close(Connection *conn);
 
 		/*
 		 * Disposes of previously-closed Connections

src/libzt.cpp

@@ -28,18 +28,31 @@
 stack driver and core ZeroTier service to create a socket-like interface
 for applications to use. See also: include/libzt.h */
 
-#include <sys/socket.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdarg.h>
-#include <sys/stat.h>
 #include <unistd.h>
 #include <errno.h>
-#include <fcntl.h>
+
-#include <sys/types.h>
+#if defined(__APPLE__)
+#include <net/ethernet.h>
+#include <net/if.h>
+#endif
+#if defined(__linux__)
+#include <netinet/ether.h>
+#include <sys/socket.h>
+#include <linux/if_packet.h>
+#include <net/ethernet.h>
+#endif
+
 #include <pthread.h>
 #include <poll.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#include <sys/socket.h>
 
 #if defined(STACK_PICO)
 #include "pico_stack.h"
@@ -84,6 +97,12 @@ namespace ZeroTier {
      */
     std::map<int, std::pair<Connection*,SocketTap*>*> fdmap;
 
+    /*
+     * 
+     */
+    std::vector<void*> vtaps;
+
+    ZeroTier::Mutex _vtaps_lock;
     ZeroTier::Mutex _multiplexer_lock;
     ZeroTier::Mutex _accepted_connection_lock;
 }
@@ -125,7 +144,7 @@ void zts_simple_start(const char *path, const char *nwid)
 void zts_stop() {
     if(zt1Service) { 
         zt1Service->terminate();
-        zt1Service->removeNets();
+        dismantleTaps();
     }
 }
 
@@ -237,11 +256,12 @@ void zts_get_ipv4_address(const char *nwid, char *addrstr, const int addrlen)
 {
     if(zt1Service) {
         uint64_t nwid_int = strtoull(nwid, NULL, 16);
-        ZeroTier::SocketTap *tap = zt1Service->getTap(nwid_int);
+        ZeroTier::SocketTap *tap = getTapByNWID(nwid_int);
         if(tap && tap->_ips.size()){ 
             for(int i=0; i<tap->_ips.size(); i++) {
                 if(tap->_ips[i].isV4()) {
-                    std::string addr = tap->_ips[i].toString();
+                	char ipbuf[64];
+                    std::string addr = tap->_ips[i].toString(ipbuf);
                     int len = addrlen < addr.length() ? addrlen : addr.length();
                     memset(addrstr, 0, len);
                     memcpy(addrstr, addr.c_str(), len); 
@@ -258,11 +278,12 @@ void zts_get_ipv6_address(const char *nwid, char *addrstr, const int addrlen)
 {
     if(zt1Service) {
         uint64_t nwid_int = strtoull(nwid, NULL, 16);
-        ZeroTier::SocketTap *tap = zt1Service->getTap(nwid_int);
+        ZeroTier::SocketTap *tap = getTapByNWID(nwid_int);
         if(tap && tap->_ips.size()){ 
             for(int i=0; i<tap->_ips.size(); i++) {
                 if(tap->_ips[i].isV6()) {
-                    std::string addr = tap->_ips[i].toString();
+                	char ipbuf[64];
+                    std::string addr = tap->_ips[i].toString(ipbuf);
                     int len = addrlen < addr.length() ? addrlen : addr.length();
                     memset(addrstr, 0, len);
                     memcpy(addrstr, addr.c_str(), len); 
@@ -279,14 +300,16 @@ void zts_get_6plane_addr(char *addr, const char *nwid, const char *devID)
 {
     ZeroTier::InetAddress _6planeAddr = ZeroTier::InetAddress::makeIpv66plane(
         ZeroTier::Utils::hexStrToU64(nwid),ZeroTier::Utils::hexStrToU64(devID));
-    memcpy(addr, _6planeAddr.toIpString().c_str(), 40);
+    char ipbuf[64];
+    memcpy(addr, _6planeAddr.toIpString(ipbuf), 40);
 }
 
 void zts_get_rfc4193_addr(char *addr, const char *nwid, const char *devID)
 {
     ZeroTier::InetAddress _6planeAddr = ZeroTier::InetAddress::makeIpv6rfc4193(
         ZeroTier::Utils::hexStrToU64(nwid),ZeroTier::Utils::hexStrToU64(devID));
-    memcpy(addr, _6planeAddr.toIpString().c_str(), 40);
+    char ipbuf[64];
+    memcpy(addr, _6planeAddr.toIpString(ipbuf), 40);
 }
 
 unsigned long zts_get_peer_count() {
@@ -361,14 +384,19 @@ int zts_socket(ZT_SOCKET_SIG) {
         errno = EPROTONOSUPPORT; // seemingly closest match
         return -1;
     }
+
+    ZeroTier::_multiplexer_lock.lock();
+
     if(socket_type == SOCK_RAW)
     {
-        // TODO: 
+        // Connection is only used to associate a socket with a SocketTap, it has no other implication
-        //  - create Connection object, handle as you please
+        ZeroTier::Connection *conn = new ZeroTier::Connection();
-        //  - we will need to hook this up to SocketTap::put and SocketTap::_handler at some point 
+        conn->socket_family = socket_family;
-        return -1;  
+        conn->socket_type = socket_type;
+        conn->protocol = protocol;
+        ZeroTier::unmap[conn->app_fd] = conn;
+        return conn->app_fd;
     }
-    ZeroTier::_multiplexer_lock.lock();
 
 #if defined(STACK_PICO)
     struct pico_socket *p;
@@ -498,11 +526,13 @@ int zts_connect(ZT_CONNECT_SIG) {
             inet_ntop(AF_INET6, 
                 (const void *)&((struct sockaddr_in6 *)addr)->sin6_addr.s6_addr, ipstr, INET6_ADDRSTRLEN);
             // TODO: This is a hack, determine a proper way to do this
-            iaddr.fromString(ipstr + std::string("/88"));
+            char addrstr[64];
+            sprintf(addrstr, "%s%s", ipstr, std::string("/88").c_str());
+            iaddr.fromString(addrstr);
             port = ((struct sockaddr_in6*)addr)->sin6_port;
         }
         DEBUG_EXTRA("fd = %d, %s : %d", fd, ipstr, ntohs(port));
-        tap = zt1Service->getTap(iaddr);
+        tap = getTapByAddr(iaddr);
         if(!tap) {
             DEBUG_ERROR("no route to host");
             errno = ENETUNREACH;
@@ -636,7 +666,7 @@ int zts_bind(ZT_BIND_SIG) {
         }
         DEBUG_EXTRA("fd = %d, %s : %d", fd, ipstr, ntohs(port));
         iaddr.fromString(ipstr);
-        tap = zt1Service->getTap(iaddr);
+        tap = getTapByAddr(iaddr);
 
         if(!tap) {
             DEBUG_ERROR("no matching interface to bind to");            
@@ -1082,7 +1112,6 @@ int zts_select(ZT_SELECT_SIG)
 
 int zts_fcntl(ZT_FCNTL_SIG)
 {
-    //DEBUG_INFO("fd = %d", fd);
     int err = 0;
     if(fd < 0) {
         errno = EBADF;
@@ -1094,7 +1123,102 @@ int zts_fcntl(ZT_FCNTL_SIG)
     return err;
 }
 
-// TODO
+/*
+       struct ifreq {
+           char ifr_name[IFNAMSIZ];
+           union {
+               struct sockaddr ifr_addr;
+               struct sockaddr ifr_dstaddr;
+               struct sockaddr ifr_broadaddr;
+               struct sockaddr ifr_netmask;
+               struct sockaddr ifr_hwaddr;
+               short           ifr_flags;
+               int             ifr_ifindex;
+               int             ifr_metric;
+               int             ifr_mtu;
+               struct ifmap    ifr_map;
+               char            ifr_slave[IFNAMSIZ];
+               char            ifr_newname[IFNAMSIZ];
+               char           *ifr_data;
+           };
+       };
+*/
+
+/*
+    [  ] [BADF]             fd is not a valid file descriptor.
+    [  ] [EFAULT]           argp references an inaccessible memory area.
+    [  ] [EINVAL]           request or argp is not valid.
+    [  ] [ENOTTY]           The specified request does not apply to the kind of object that the file descriptor fd references.
+*/
+
+int zts_ioctl(ZT_IOCTL_SIG)
+{
+    int err = 0;
+    if(fd < 0) {
+        errno = EBADF;
+        err = -1;
+    }
+    else {
+#if defined(__linux__)
+        if(argp)
+        {
+            struct ifreq *ifr = (struct ifreq *)argp;
+            ZeroTier::SocketTap *tap = getTapByName(ifr->ifr_name);
+            if(!tap) {
+                DEBUG_ERROR("unable to locate tap interface with that name");
+                err = -1;
+                errno = EINVAL;
+            }
+            // index of SocketTap interface
+            if(request == SIOCGIFINDEX) {
+                ifr->ifr_ifindex = tap->ifindex;
+                err = 0;
+            }
+            // MAC addres or SocketTap
+            if(request == SIOCGIFHWADDR) {
+                tap->_mac.copyTo(&(ifr->ifr_hwaddr.sa_data), sizeof(ifr->ifr_hwaddr.sa_data));
+                err = 0;
+            }
+            // IP address of SocketTap
+            if(request == SIOCGIFADDR) {
+                struct sockaddr_in *in4 = (struct sockaddr_in *)&(ifr->ifr_addr);
+                memcpy(&(in4->sin_addr.s_addr), tap->_ips[0].rawIpData(), sizeof(ifr->ifr_addr));
+                err = 0;
+            }
+        }
+        else
+        {
+            DEBUG_INFO("!argp");
+        }
+#else
+        err = ioctl(fd, request, argp);
+#endif
+    }
+    return err;
+}
+
+
+/*
+
+Linux:
+
+    [  ] [EAGAIN or EWOULDBLOCK] The socket is marked nonblocking and the requested operation would block. POSIX.1-2001 allows either error to be returned for this case, and does not require these constants to have the same value, so a portable application should check for both possibilities.
+    [  ] [EBADF]                 An invalid descriptor was specified.
+    [  ] [ECONNRESET]            Connection reset by peer.
+    [  ] [EDESTADDRREQ]          The socket is not connection-mode, and no peer address is set.
+    [  ] [EFAULT]                An invalid user space address was specified for an argument.
+    [  ] [EINTR]                 A signal occurred before any data was transmitted; see signal(7).
+    [  ] [EINVAL]                Invalid argument passed.
+    [  ] [EISCONN]               The connection-mode socket was connected already but a recipient was specified. (Now either this error is returned, or the recipient specification is ignored.)
+    [  ] [EMSGSIZE]              The socket type requires that message be sent atomically, and the size of the message to be sent made this impossible.
+    [  ] [ENOBUFS]               The output queue for a network interface was full. This generally indicates that the interface has stopped sending, but may be caused by transient congestion. (Normally, this does not occur in Linux. Packets are just silently dropped when a device queue overflows.)
+    [  ] [ENOMEM]                No memory available.
+    [  ] [ENOTCONN]              The socket is not connected, and no target has been given.
+    [  ] [ENOTSOCK]              The argument sockfd is not a socket.
+    [  ] [EOPNOTSUPP]            Some bit in the flags argument is inappropriate for the socket type.
+    [  ] [EPIPE]                 The local end has been shut down on a connection oriented socket. In this case the process will also receive a SIGPIPE unless MSG_NOSIGNAL is set.
+
+*/
 ssize_t zts_sendto(ZT_SENDTO_SIG)
 {
     DEBUG_INFO("fd = %d", fd);
@@ -1104,7 +1228,29 @@ ssize_t zts_sendto(ZT_SENDTO_SIG)
         err = -1;
     }
     else {
-        err = sendto(fd, buf, len, flags, addr, addrlen);
+        struct sockaddr_ll *socket_address = (struct sockaddr_ll *)addr;
+        ZeroTier::SocketTap *tap = getTapByIndex(socket_address->sll_ifindex);
+        if(tap)
+        {
+            DEBUG_INFO("found interface of ifindex=%d", tap->ifindex);
+            ZeroTier::Connection *conn = ZeroTier::unmap[fd];
+            if(conn) {
+                DEBUG_INFO("located connection object for fd=%d", fd);
+                err = tap->Write(conn, (void*)buf, len);
+            }
+            else {
+                DEBUG_ERROR("unable to locate connection object for fd=%d", fd);
+                err = -1;
+                errno = EINVAL;
+            }
+        }
+        else
+        {
+            DEBUG_ERROR("unable to locate tap of ifindex=%d", socket_address->sll_ifindex);
+            err = -1;
+            errno = EINVAL;
+        }
+        //err = sendto(fd, buf, len, flags, addr, addrlen);
     }
     return err;
 }
@@ -1566,6 +1712,78 @@ int zts_maxsockets()
     return 32;
 }
 
+/****************************************************************************/
+/* ZeroTier Core helper functions for libzt - DON'T CALL THESE DIRECTLY     */
+/****************************************************************************/
+
+ZeroTier::SocketTap *getTapByNWID(uint64_t nwid)
+{
+    ZeroTier::_vtaps_lock.lock();
+    ZeroTier::SocketTap *s, *tap = nullptr;    
+    for(int i=0; i<ZeroTier::vtaps.size(); i++) {
+        s = (ZeroTier::SocketTap*)ZeroTier::vtaps[i];
+        if(s->_nwid == nwid) { tap = s; }
+    }
+    ZeroTier::_vtaps_lock.unlock();
+    return tap;
+}
+
+ZeroTier::SocketTap *getTapByAddr(ZeroTier::InetAddress &addr)
+{
+    ZeroTier::_vtaps_lock.lock();
+    ZeroTier::SocketTap *s, *tap = nullptr; 
+    for(int i=0; i<ZeroTier::vtaps.size(); i++) {
+        s = (ZeroTier::SocketTap*)ZeroTier::vtaps[i];
+        for(int j=0; j<s->_ips.size(); j++) {
+            if(s->_ips[j].isEqualPrefix(addr) 
+                || s->_ips[j].ipsEqual(addr) 
+                || s->_ips[j].containsAddress(addr)) 
+            {
+                tap = s;
+            }
+        }
+    }
+    ZeroTier::_vtaps_lock.unlock();
+    return tap;
+}
+
+ZeroTier::SocketTap *getTapByName(char *ifname)
+{
+    ZeroTier::_vtaps_lock.lock();
+    ZeroTier::SocketTap *s, *tap = nullptr;  
+    for(int i=0; i<ZeroTier::vtaps.size(); i++) {
+        s = (ZeroTier::SocketTap*)ZeroTier::vtaps[i];
+        if(!strcmp(s->_dev.c_str(), ifname)) {
+            tap = s;
+        }
+    }
+    ZeroTier::_vtaps_lock.unlock();
+    return tap;
+}
+
+ZeroTier::SocketTap *getTapByIndex(int index)
+{
+    ZeroTier::_vtaps_lock.lock();
+    ZeroTier::SocketTap *s, *tap = nullptr;    
+    for(int i=0; i<ZeroTier::vtaps.size(); i++) {
+        s = (ZeroTier::SocketTap*)ZeroTier::vtaps[i];
+        if(s->ifindex == index) {
+            tap = s;
+        }
+    }
+    ZeroTier::_vtaps_lock.unlock();
+    return tap;
+}
+
+void dismantleTaps()
+{
+    ZeroTier::_vtaps_lock.lock();
+    for(int i=0; i<ZeroTier::vtaps.size(); i++) { delete (ZeroTier::SocketTap*)ZeroTier::vtaps[i]; }
+    ZeroTier::vtaps.clear();
+    ZeroTier::_vtaps_lock.unlock();
+}
+
+
 // Starts a ZeroTier service in the background
 void *zts_start_service(void *thread_id) {
 

src/lwIP.hpp

@@ -55,6 +55,7 @@ struct netif;
     #define LWIP_ETHARP_OUTPUT_SIG struct netif *netif, struct pbuf *q, const ip4_addr_t *ipaddr
 #endif
 #if defined(LIBZT_IPV6)
+#include "lwip/ip6_addr.h"
     #define LWIP_NETIF_ADD_SIG struct netif *netif, void *state, netif_init_fn init, netif_input_fn input
     #define LWIP_ETHIP6_OUTPUT_SIG struct netif *netif, struct pbuf *q, const ip6_addr_t *ip6addr
     #define LWIP_ETHARP_OUTPUT_SIG struct netif *netif, struct pbuf *q, const ip6_addr_t *ipaddr

src/picoTCP.cpp

@@ -85,6 +85,7 @@ namespace ZeroTier {
 
 	bool picoTCP::pico_init_interface(SocketTap *tap, const InetAddress &ip)
 	{
+		char ipbuf[64];
 		if (std::find(tap->_ips.begin(),tap->_ips.end(),ip) == tap->_ips.end()) {
 			tap->_ips.push_back(ip);
 			std::sort(tap->_ips.begin(),tap->_ips.end());
@@ -97,7 +98,7 @@ namespace ZeroTier {
 			    picodev.tap = tap;
 			    uint8_t mac[PICO_SIZE_ETH];
 			    tap->_mac.copyTo(mac, PICO_SIZE_ETH);
-			    if(pico_device_init(&picodev, "pz", mac) != 0) {
+			    if(pico_device_init(&picodev, tap->_dev.c_str(), mac) != 0) {
 			        DEBUG_ERROR("dev init failed");
 			        handle_general_failure();
 			        return false;
@@ -110,7 +111,8 @@ namespace ZeroTier {
 			    ipaddr.addr = *((uint32_t *)ip.rawIpData());
 			    netmask.addr = *((uint32_t *)ip.netmask().rawIpData());
 			    pico_ipv4_link_add(&picodev, ipaddr, netmask);
-			    DEBUG_INFO("addr  = %s", ip.toString().c_str());
+			    DEBUG_INFO("addr  = %s", ip.toString(ipbuf));
+			    //DEBUG_INFO("mac   = %s", tap->_mac.toString(ipbuf));
 			    return true;
 			}
 			if(ip.isV6())
@@ -123,6 +125,7 @@ namespace ZeroTier {
 		    	pico_string_to_ipv6(nm_str, netmask.addr);
 			    pico_ipv6_link_add(&picodev, ipaddr, netmask);
 			    DEBUG_INFO("addr6 = %s", ipv6_str);
+			    //DEBUG_INFO("mac   = %s", tap->_mac.toString(ipbuf));
 			   	return true;    
 			}
 		}
@@ -686,7 +689,7 @@ namespace ZeroTier {
     	return new_conn;
     }
 
-    void picoTCP::pico_Read(SocketTap *tap, PhySocket *sock, Connection* conn, bool stack_invoked)
+    int picoTCP::pico_Read(SocketTap *tap, PhySocket *sock, Connection* conn, bool stack_invoked)
     {
     	DEBUG_INFO();
     	//exit(0);
@@ -766,26 +769,28 @@ namespace ZeroTier {
         }
         // DEBUG_FLOW("[ ZTSOCK <- RXBUF] Emitted (%d) from RXBUF(%d) to socket", tot, conn->rxsz);
         */
+        return 0;
     }
 
-    void picoTCP::pico_Write(Connection *conn, void *data, ssize_t len)
+    int picoTCP::pico_Write(Connection *conn, void *data, ssize_t len)
     {
+    	int err = 0;
 	    // TODO: Add RingBuffer overflow checks
 	    //DEBUG_INFO("conn=%p, len = %d", conn, len);
 	    Mutex::Lock _l(conn->_tx_m);
 		if(len <= 0) {
     		DEBUG_ERROR("invalid write length (len=%d)", len);
     		handle_general_failure();
-    		return;
+    		return -1;
     	}
     	if(conn->picosock->state & PICO_SOCKET_STATE_CLOSED){
     		DEBUG_ERROR("socket is CLOSED, this write() will fail");
-    		return;
+    		return -1;
     	}
     	if(!conn) {
     		DEBUG_ERROR("invalid connection (len=%d)", len);
     		handle_general_failure();
-    		return;
+    		return -1;
     	}
 
     	int original_txsz = conn->TXbuf->count();
@@ -798,7 +803,7 @@ namespace ZeroTier {
 
 	    int buf_w = conn->TXbuf->write((const unsigned char*)data, len);
             if (buf_w != len) {
-		// because we checked ZT_TCP_TX_BUF_SZ above, this should not happen
+		    // because we checked ZT_TCP_TX_BUF_SZ above, this should not happen
     		DEBUG_ERROR("TX wrote only %d but expected to write %d", buf_w, len);
     		exit(0);
 	    }
@@ -813,7 +818,10 @@ namespace ZeroTier {
 	    
 	    if((r = pico_socket_write(conn->picosock, conn->TXbuf->get_buf(), max_write_len)) < 0) {
 	    	DEBUG_ERROR("unable to write to picosock=%p, r=%d", conn->picosock, r);
-	    	return;
+	    	err = -1;
+	    }
+	    else {
+	    	err = r; // successful write
 	    }
 	   	if(conn->socket_type == SOCK_STREAM) {
 	    	//DEBUG_TRANS("[ TCP TX -> STACK] :: conn = %p, len = %d", conn, r);
@@ -823,6 +831,8 @@ namespace ZeroTier {
 	    }
 	    if(r>0)
 	    	conn->TXbuf->consume(r);
+
+	    return err;
     }
 
     int picoTCP::pico_Close(Connection *conn)

src/picoTCP.hpp

@@ -40,7 +40,7 @@
 #include "SocketTap.hpp"
 
 /****************************************************************************/
-/* PicoTCP API Signatures (See libzt.h for the API an app should use) */
+/* PicoTCP API Signatures (See libzt.h for the API an app should use)       */
 /****************************************************************************/
 
 #define PICO_IPV4_TO_STRING_SIG char *ipbuf, const uint32_t ip
@@ -150,12 +150,12 @@ namespace ZeroTier
     	/*
 		 * Read from RX buffer to application - Called from SocketTap
 		 */
-    	void pico_Read(SocketTap *tap, ZeroTier::PhySocket *sock,Connection *conn,bool stack_invoked);
+    	int pico_Read(SocketTap *tap, ZeroTier::PhySocket *sock,Connection *conn,bool stack_invoked);
 
     	/*
 		 * Write to userspace network stack - Called from SocketTap
 		 */
-    	void pico_Write(Connection *conn, void *data, ssize_t len);
+    	int pico_Write(Connection *conn, void *data, ssize_t len);
 
     	/*
 		 * Close a Connection - Called from SocketTap

test/layer2.cpp

@@ -1,13 +1,40 @@
 // This file is built with libzt.a via `make tests`
 
-#include <unistd.h>
 #include <stdio.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <netinet/in.h>
 #include <sys/socket.h>
+#include <arpa/inet.h>
+#include <net/if.h>
 #include <netinet/ip.h>
 #include <netinet/udp.h>
 
+#if defined(__APPLE__)
+#include <net/ethernet.h>
+#endif
+#if defined(__linux__)
+#include <netinet/ether.h>
+#include <sys/socket.h>
+#include <linux/if_packet.h>
+#include <net/ethernet.h>
+#endif
+
 #include "libzt.h"
 
+unsigned short csum(unsigned short *buf, int nwords)
+{
+    unsigned long sum;
+    for(sum=0; nwords>0; nwords--)
+        sum += *buf++;
+    sum = (sum >> 16) + (sum &0xffff);
+    sum += (sum >> 16);
+    return (unsigned short)(~sum);
+}
+
 int main(int argc , char *argv[])
 {
 	if(argc < 3) {
@@ -15,19 +42,150 @@ int main(int argc , char *argv[])
         return 1;
     }
 
+    // initialize library
 	printf("Starting libzt...\n");
 	zts_simple_start(argv[1], argv[2]);
 	char device_id[11];
 	zts_get_device_id(device_id);
-	printf("Complete. I am %s\n", device_id);
+	fprintf(stderr, "Complete. I am %s\n", device_id);
 
-	// layer2 example code
+	// create socket
-	int rawsock;
+	int fd;
-	if((rawsock = zts_socket(AF_INET, SOCK_RAW, IPPROTO_UDP)) < 0)
+	if((fd = zts_socket(AF_INET, SOCK_RAW, IPPROTO_UDP)) < 0) {
-	{
 		printf("There was a problem creating the raw socket\n");
-		return -1;
+		exit(-1);
 	}
-	printf("Created raw socket (%d)\n", rawsock);
+	fprintf(stderr, "Created raw socket (%d)\n", fd);
+
+#if defined(__linux__) // The rest of this file isn't yet supported on non-linux platforms
+	// get interface index to bind on
+	struct ifreq if_idx;
+	memset(&if_idx, 0, sizeof(struct ifreq));
+	strncpy(if_idx.ifr_name, "libzt0", IFNAMSIZ-1);
+	if (zts_ioctl(fd, SIOCGIFINDEX, &if_idx) < 0) {
+		perror("SIOCGIFINDEX");
+		exit(-1);
+	}
+	fprintf(stderr, "if_idx.ifr_ifindex=%d\n", if_idx.ifr_ifindex);
+
+	// get MAC address of interface to send on
+	struct ifreq if_mac;
+	memset(&if_mac, 0, sizeof(struct ifreq));
+	strncpy(if_mac.ifr_name, "libzt0", IFNAMSIZ-1);
+	if (zts_ioctl(fd, SIOCGIFHWADDR, &if_mac) < 0) {
+		perror("SIOCGIFHWADDR");
+		exit(-1);
+	}
+	const unsigned char* mac=(unsigned char*)if_mac.ifr_hwaddr.sa_data;
+	fprintf(stderr, "hwaddr=%02X:%02X:%02X:%02X:%02X:%02X\n", mac[0],mac[1],mac[2],mac[3],mac[4],mac[5]);
+
+	// get IP address of interface to send on
+	struct ifreq if_ip;
+	memset(&if_ip, 0, sizeof(struct ifreq));
+	strncpy(if_ip.ifr_name, "libzt0", IFNAMSIZ-1);
+	if (zts_ioctl(fd, SIOCGIFADDR, &if_ip) < 0) {
+		perror("SIOCGIFADDR");
+		exit(-1);
+	}
+	char ipv4_str[INET_ADDRSTRLEN];
+	struct sockaddr_in *in4 = (struct sockaddr_in *)&if_ip.ifr_addr;
+	inet_ntop(AF_INET, (const void *)&in4->sin_addr.s_addr, ipv4_str, INET_ADDRSTRLEN);
+	fprintf(stderr, "addr=%s", ipv4_str);
+
+	// construct ethernet header
+	int tx_len = 0;
+	char sendbuf[1024];
+	struct ether_header *eh = (struct ether_header *) sendbuf;
+	memset(sendbuf, 0, 1024);
+	
+	// Ethernet header
+	eh->ether_shost[0] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[0];
+	eh->ether_shost[1] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[1];
+	eh->ether_shost[2] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[2];
+	eh->ether_shost[3] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[3];
+	eh->ether_shost[4] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[4];
+	eh->ether_shost[5] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[5];
+
+	// set destination MAC
+	int MY_DEST_MAC0 = 0x72;
+	int MY_DEST_MAC1 = 0x92;
+	int MY_DEST_MAC2 = 0xd4;
+	int MY_DEST_MAC3 = 0xfd;
+	int MY_DEST_MAC4 = 0x43;
+	int MY_DEST_MAC5 = 0x45;
+
+	eh->ether_dhost[0] = MY_DEST_MAC0;
+	eh->ether_dhost[1] = MY_DEST_MAC1;
+	eh->ether_dhost[2] = MY_DEST_MAC2;
+	eh->ether_dhost[3] = MY_DEST_MAC3;
+	eh->ether_dhost[4] = MY_DEST_MAC4;
+	eh->ether_dhost[5] = MY_DEST_MAC5;
+	eh->ether_type = htons(ETH_P_IP);
+	tx_len += sizeof(struct ether_header);
+
+
+	// Construct the IP header
+	int ttl = 64;
+	struct iphdr *iph = (struct iphdr *) (sendbuf + sizeof(struct ether_header));
+	iph->ihl = 5;
+	iph->version = 4;
+	iph->tos = 16; // Low delay
+	iph->id = htons(54321);
+	iph->ttl = ttl; // hops
+	iph->protocol = 17; // UDP
+	// Source IP address, can be spoofed
+	iph->saddr = inet_addr(inet_ntoa(((struct sockaddr_in *)&if_ip.ifr_addr)->sin_addr));
+	// iph->saddr = inet_addr("192.168.0.112");
+	// Destination IP address
+	iph->daddr = inet_addr("10.7.7.1");
+	tx_len += sizeof(struct iphdr);
+
+	// Construct UDP header
+	struct udphdr *udph = (struct udphdr *) (sendbuf + sizeof(struct iphdr) + sizeof(struct ether_header));
+	udph->source = htons(3423);
+	udph->dest = htons(5342);
+	udph->check = 0; // skip
+	tx_len += sizeof(struct udphdr);
+
+	// Fill in UDP payload
+	sendbuf[tx_len++] = 0xde;
+	sendbuf[tx_len++] = 0xad;
+	sendbuf[tx_len++] = 0xbe;
+	sendbuf[tx_len++] = 0xef;
+
+	// Fill in remaining header info
+	// Length of UDP payload and header
+	udph->len = htons(tx_len - sizeof(struct ether_header) - sizeof(struct iphdr));
+	// Length of IP payload and header
+	iph->tot_len = htons(tx_len - sizeof(struct ether_header));
+	// Calculate IP checksum on completed header
+	iph->check = csum((unsigned short *)(sendbuf+sizeof(struct ether_header)), sizeof(struct iphdr)/2);
+
+	// Send packet
+	// Destination address
+	struct sockaddr_ll socket_address;
+	// Index of the network device
+	socket_address.sll_ifindex = if_idx.ifr_ifindex;
+	// Address length
+	socket_address.sll_halen = ETH_ALEN;
+	// Destination MAC
+	socket_address.sll_addr[0] = MY_DEST_MAC0;
+	socket_address.sll_addr[1] = MY_DEST_MAC1;
+	socket_address.sll_addr[2] = MY_DEST_MAC2;
+	socket_address.sll_addr[3] = MY_DEST_MAC3;
+	socket_address.sll_addr[4] = MY_DEST_MAC4;
+	socket_address.sll_addr[5] = MY_DEST_MAC5;
+
+	while(1)
+	{
+		usleep(10000);
+		// Send packet
+		if (zts_sendto(fd, sendbuf, tx_len, 0, (struct sockaddr*)&socket_address, sizeof(struct sockaddr_ll)) < 0)
+		    fprintf(stderr, "Send failed\n");
+	}
+
+	// dismantle all zt virtual taps (SocketTaps)
+	zts_stop();
+#endif
 	return 0;
 }