From 307d16493863b6c8e13e74923d6abec2cd9e8009 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 4 May 2017 15:33:33 -0700 Subject: [PATCH] updated core to 1.2.4, fixed pico_device init bug --- include/ZeroTierSDK.h | 4 + make-mac.mk | 1 + src/SocketTap.cpp | 2 +- src/SocketTap.hpp | 6 +- src/picoTCP.cpp | 37 +- zto/Jenkinsfile | 82 + zto/LICENSE.txt | 37 + zto/RELEASE-NOTES.md | 14 +- zto/controller/EmbeddedNetworkController.cpp | 648 +- zto/controller/EmbeddedNetworkController.hpp | 113 +- zto/controller/JSONDB.cpp | 502 +- zto/controller/JSONDB.hpp | 127 +- zto/controller/README.md | 22 +- zto/controller/migrate-sqlite/migrate.js | 2 - zto/ext/arm32-neon-salsa2012-asm/README.md | 6 + zto/ext/arm32-neon-salsa2012-asm/salsa2012.h | 23 + zto/ext/arm32-neon-salsa2012-asm/salsa2012.s | 2231 +++++++ .../linux/zerotier-containerized/Dockerfile | 20 + .../linux/zerotier-containerized/main.sh | 10 + zto/ext/installfiles/mac/ZeroTier One.pkgproj | 2 +- zto/ext/installfiles/mac/postinst.sh | 9 + zto/ext/installfiles/mac/preinst.sh | 13 + zto/ext/installfiles/mac/uninstall.sh | 12 +- zto/ext/installfiles/windows/ZeroTier One.aip | 8 +- .../zerotier-one/zerotier-one.nuspec | 2 +- zto/ext/json/LICENSE.MIT | 17 +- zto/ext/json/README.md | 418 +- zto/ext/json/json.hpp | 5592 +++++++++++------ zto/ext/x64-salsa2012-asm/README.md | 6 + zto/ext/x64-salsa2012-asm/salsa2012.h | 16 + zto/ext/x64-salsa2012-asm/salsa2012.s | 4488 +++++++++++++ zto/include/ZeroTierOne.h | 10 +- zto/make-bsd.mk | 80 +- zto/make-linux.mk | 100 +- zto/make-mac.mk | 7 + zto/node/Address.hpp | 10 +- zto/node/Array.hpp | 10 +- zto/node/AtomicCounter.hpp | 10 +- zto/node/Buffer.hpp | 10 +- zto/node/C25519.cpp | 4 +- zto/node/C25519.hpp | 10 +- zto/node/Capability.cpp | 10 +- zto/node/Capability.hpp | 15 +- zto/node/CertificateOfMembership.cpp | 10 +- zto/node/CertificateOfMembership.hpp | 35 +- zto/node/CertificateOfOwnership.cpp | 10 +- zto/node/CertificateOfOwnership.hpp | 17 +- zto/node/CertificateOfRepresentation.hpp | 16 +- zto/node/Cluster.cpp | 10 +- zto/node/Cluster.hpp | 10 +- zto/node/Constants.hpp | 12 +- zto/node/Credential.hpp | 66 + zto/node/Dictionary.hpp | 70 +- zto/node/Hashtable.hpp | 10 +- zto/node/Identity.cpp | 12 +- zto/node/Identity.hpp | 10 +- zto/node/IncomingPacket.cpp | 46 +- zto/node/IncomingPacket.hpp | 10 +- zto/node/InetAddress.cpp | 12 +- zto/node/InetAddress.hpp | 18 +- zto/node/MAC.hpp | 10 +- zto/node/Membership.cpp | 306 +- zto/node/Membership.hpp | 260 +- zto/node/MulticastGroup.hpp | 10 +- zto/node/Multicaster.cpp | 10 +- zto/node/Multicaster.hpp | 10 +- zto/node/Mutex.hpp | 10 +- zto/node/Network.cpp | 52 +- zto/node/Network.hpp | 17 +- zto/node/NetworkConfig.cpp | 10 +- zto/node/NetworkConfig.hpp | 10 +- zto/node/NetworkController.hpp | 10 +- zto/node/Node.cpp | 74 +- zto/node/Node.hpp | 18 +- zto/node/NonCopyable.hpp | 10 +- zto/node/OutboundMulticast.cpp | 10 +- zto/node/OutboundMulticast.hpp | 10 +- zto/node/Packet.cpp | 126 +- zto/node/Packet.hpp | 10 +- zto/node/Path.cpp | 10 +- zto/node/Path.hpp | 10 +- zto/node/Peer.cpp | 299 +- zto/node/Peer.hpp | 148 +- zto/node/Poly1305.cpp | 21 +- zto/node/Poly1305.hpp | 10 +- zto/node/Revocation.cpp | 10 +- zto/node/Revocation.hpp | 51 +- zto/node/RuntimeEnvironment.hpp | 10 +- zto/node/SHA512.cpp | 35 +- zto/node/SHA512.hpp | 10 +- zto/node/Salsa20.cpp | 57 +- zto/node/Salsa20.hpp | 98 +- zto/node/SelfAwareness.cpp | 10 +- zto/node/SelfAwareness.hpp | 10 +- zto/node/SharedPtr.hpp | 10 +- zto/node/Switch.cpp | 10 +- zto/node/Switch.hpp | 10 +- zto/node/Tag.cpp | 10 +- zto/node/Tag.hpp | 29 +- zto/node/Topology.cpp | 10 +- zto/node/Topology.hpp | 14 +- zto/node/Utils.cpp | 14 +- zto/node/Utils.hpp | 10 +- zto/node/World.hpp | 10 +- zto/one.cpp | 12 +- zto/osdep/Arp.cpp | 10 +- zto/osdep/Arp.hpp | 10 +- zto/osdep/BSDEthernetTap.cpp | 10 +- zto/osdep/BSDEthernetTap.hpp | 10 +- zto/osdep/Binder.hpp | 10 +- zto/osdep/BlockingQueue.hpp | 37 +- zto/osdep/Http.cpp | 47 +- zto/osdep/Http.hpp | 43 +- zto/osdep/LinuxEthernetTap.cpp | 57 +- zto/osdep/LinuxEthernetTap.hpp | 10 +- zto/osdep/ManagedRoute.cpp | 75 +- zto/osdep/ManagedRoute.hpp | 26 + zto/osdep/NeighborDiscovery.cpp | 10 +- zto/osdep/NeighborDiscovery.hpp | 10 +- zto/osdep/OSUtils.cpp | 16 +- zto/osdep/OSUtils.hpp | 15 +- zto/osdep/OSXEthernetTap.cpp | 62 +- zto/osdep/OSXEthernetTap.hpp | 10 +- zto/osdep/Phy.hpp | 12 +- zto/osdep/PortMapper.cpp | 10 +- zto/osdep/PortMapper.hpp | 10 +- zto/osdep/TestEthernetTap.hpp | 156 + zto/osdep/Thread.hpp | 52 +- zto/osdep/WindowsEthernetTap.cpp | 10 +- zto/osdep/WindowsEthernetTap.hpp | 10 +- zto/selftest.cpp | 218 +- zto/service/ClusterDefinition.hpp | 10 +- zto/service/ClusterGeoIpService.cpp | 10 +- zto/service/ClusterGeoIpService.hpp | 10 +- zto/service/OneService.cpp | 81 +- zto/service/OneService.hpp | 17 +- zto/service/SoftwareUpdater.cpp | 10 +- zto/service/SoftwareUpdater.hpp | 10 +- zto/tcp-proxy/Makefile | 7 - zto/tcp-proxy/README.md | 4 - zto/tcp-proxy/tcp-proxy.cpp | 317 - zto/version.h | 12 +- zto/zerotier-one.spec | 5 +- 143 files changed, 14284 insertions(+), 4176 deletions(-) create mode 100644 zto/Jenkinsfile create mode 100644 zto/LICENSE.txt create mode 100644 zto/ext/arm32-neon-salsa2012-asm/README.md create mode 100644 zto/ext/arm32-neon-salsa2012-asm/salsa2012.h create mode 100644 zto/ext/arm32-neon-salsa2012-asm/salsa2012.s create mode 100644 zto/ext/installfiles/linux/zerotier-containerized/Dockerfile create mode 100755 zto/ext/installfiles/linux/zerotier-containerized/main.sh create mode 100644 zto/ext/x64-salsa2012-asm/README.md create mode 100644 zto/ext/x64-salsa2012-asm/salsa2012.h create mode 100644 zto/ext/x64-salsa2012-asm/salsa2012.s create mode 100644 zto/node/Credential.hpp create mode 100644 zto/osdep/TestEthernetTap.hpp delete mode 100644 zto/tcp-proxy/Makefile delete mode 100644 zto/tcp-proxy/README.md delete mode 100644 zto/tcp-proxy/tcp-proxy.cpp diff --git a/include/ZeroTierSDK.h b/include/ZeroTierSDK.h index 721d266..80aa0a8 100644 --- a/include/ZeroTierSDK.h +++ b/include/ZeroTierSDK.h @@ -39,6 +39,10 @@ #define ZT_SDK_RPC_DIR_PREFIX "rpc.d" +#define ZT_CORE_VERSION_MAJOR 1 +#define ZT_CORE_VERSION_MINOR 2 +#define ZT_CORE_VERSION_REVISION 4 + #define ZT_SDK_VERSION_MAJOR 1 #define ZT_SDK_VERSION_MINOR 0 #define ZT_SDK_VERSION_REVISION 0 diff --git a/make-mac.mk b/make-mac.mk index eae2253..6b86687 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -56,6 +56,7 @@ INCLUDES+= -Iext \ -I$(ZTO)/node \ -I$(ZTO)/service \ -I$(ZTO)/include \ + -I$(ZTO)/controller \ -I../$(ZTO)/osdep \ -I../$(ZTO)/node \ -I../$(ZTO)/service \ diff --git a/src/SocketTap.cpp b/src/SocketTap.cpp index 41bfcf5..39351b0 100644 --- a/src/SocketTap.cpp +++ b/src/SocketTap.cpp @@ -292,7 +292,7 @@ namespace ZeroTier { const struct sockaddr *from) {} void SocketTap::phyOnTcpClose(PhySocket *sock,void **uptr) {} void SocketTap::phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) {} - void SocketTap::phyOnTcpWritable(PhySocket *sock,void **uptr, bool stack_invoked) {} + void SocketTap::phyOnTcpWritable(PhySocket *sock,void **uptr) {} } // namespace ZeroTier diff --git a/src/SocketTap.hpp b/src/SocketTap.hpp index 1b45c18..3e40dde 100644 --- a/src/SocketTap.hpp +++ b/src/SocketTap.hpp @@ -135,8 +135,8 @@ namespace ZeroTier { /* Vars */ /****************************************************************************/ - struct pico_device picodev; - struct pico_device picodev6; + struct pico_device *picodev; + struct pico_device *picodev6; std::vector ips() const; std::vector _ips; @@ -236,7 +236,7 @@ namespace ZeroTier { const struct sockaddr *from); void phyOnTcpClose(PhySocket *sock,void **uptr); void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len); - void phyOnTcpWritable(PhySocket *sock,void **uptr, bool stack_invoked); + void phyOnTcpWritable(PhySocket *sock,void **uptr); }; } // namespace ZeroTier diff --git a/src/picoTCP.cpp b/src/picoTCP.cpp index e0295a0..66d9a76 100644 --- a/src/picoTCP.cpp +++ b/src/picoTCP.cpp @@ -81,48 +81,47 @@ namespace ZeroTier { std::sort(tap->_ips.begin(),tap->_ips.end()); if(ip.isV4()) { + tap->picodev = new struct pico_device; struct pico_ip4 ipaddr, netmask; ipaddr.addr = *((uint32_t *)ip.rawIpData()); netmask.addr = *((uint32_t *)ip.netmask().rawIpData()); + tap->picodev->send = pico_eth_send; // tx + tap->picodev->poll = pico_eth_poll; // rx + tap->picodev->mtu = tap->_mtu; + tap->picodev->tap = tap; uint8_t mac[PICO_SIZE_ETH]; tap->_mac.copyTo(mac, PICO_SIZE_ETH); - // DEBUG_ATTN("mac = %s", tap->_mac.toString().c_str()); - tap->picodev.send = pico_eth_send; // tx - tap->picodev.poll = pico_eth_poll; // rx - DEBUG_INFO("tap->picodev.poll = %p", tap->picodev.poll); - tap->picodev.mtu = tap->_mtu; - tap->picodev.tap = tap; - if(pico_device_init(&(tap->picodev), "p4", mac) != 0) { + if(pico_device_init(tap->picodev, "p4", mac) != 0) { DEBUG_ERROR("dev init failed"); + delete tap->picodev; return false; } - pico_ipv4_link_add(&(tap->picodev), ipaddr, netmask); + pico_ipv4_link_add(tap->picodev, ipaddr, netmask); DEBUG_INFO("addr = %s", ip.toString().c_str()); return true; } if(ip.isV6()) { - struct pico_ip6 ipaddr, netmask; char ipv6_str[INET6_ADDRSTRLEN], nm_str[INET6_ADDRSTRLEN]; inet_ntop(AF_INET6, ip.rawIpData(), ipv6_str, INET6_ADDRSTRLEN); inet_ntop(AF_INET6, ip.netmask().rawIpData(), nm_str, INET6_ADDRSTRLEN); - DEBUG_ERROR("ipv6_str = %s", ipv6_str); - DEBUG_ERROR("nm_str = %s", nm_str); + + tap->picodev6 = new struct pico_device; + struct pico_ip6 ipaddr, netmask; pico_string_to_ipv6(ipv6_str, ipaddr.addr); pico_string_to_ipv6(nm_str, netmask.addr); - pico_ipv6_link_add(&(tap->picodev6), ipaddr, netmask); - tap->picodev6.send = pico_eth_send; // tx - tap->picodev6.poll = pico_eth_poll; // rx - tap->picodev6.tap = tap; + tap->picodev6->send = pico_eth_send; // tx + tap->picodev6->poll = pico_eth_poll; // rx + tap->picodev6->mtu = tap->_mtu; + tap->picodev6->tap = tap; uint8_t mac[PICO_SIZE_ETH]; tap->_mac.copyTo(mac, PICO_SIZE_ETH); - //mac[3]=9; - //mac[0]='b'; - // DEBUG_ATTN("mac = %s", tap->_mac.toString().c_str()); - if(pico_device_init(&(tap->picodev6), "p6", mac) != 0) { + if(pico_device_init(tap->picodev6, "p6", mac) != 0) { DEBUG_ERROR("dev init failed"); + delete tap->picodev6; return false; } + pico_ipv6_link_add(tap->picodev6, ipaddr, netmask); DEBUG_INFO("addr6 = %s", ip.toString().c_str()); return true; } diff --git a/zto/Jenkinsfile b/zto/Jenkinsfile new file mode 100644 index 0000000..74c8624 --- /dev/null +++ b/zto/Jenkinsfile @@ -0,0 +1,82 @@ +#!/usr/bin/env groovy + +node('master') { + def changelog = getChangeLog currentBuild + + slackSend "Building ${env.JOB_NAME} #${env.BUILD_NUMBER} \n Change Log: \n ${changelog}" +} + +parallel 'centos7': { + node('centos7') { + try { + checkout scm + + stage('Build Centos 7') { + sh 'make -f make-linux.mk' + } + } + catch (err) { + currentBuild.result = "FAILURE" + slackSend color: '#ff0000', message: "${env.JOB_NAME} broken on Centos 7 (<${env.BUILD_URL}|Open>)" + + throw err + } + } +}, 'android-ndk': { + node('android-ndk') { + try { + checkout scm + + stage('Build Android NDK') { + sh "/android/android-ndk-r13b/ndk-build -C $WORKSPACE/java ZT1=${WORKSPACE}" + } + } + catch (err) { + currentBuild.result = "FAILURE" + slackSend color: '#ff0000', message: "${env.JOB_NAME} broken on Android NDK (<${env.BUILD_URL}|Open>)" + + throw err + } + } +}, 'macOS': { + node('macOS') { + try { + checkout scm + + stage('Build macOS') { + sh 'make -f make-mac.mk' + } + + stage('Build macOS UI') { + sh 'cd macui && xcodebuild -target "ZeroTier One" -configuration Debug' + } + } + catch (err) { + currentBuild.result = "FAILURE" + slackSend color: '#ff0000', message: "${env.JOB_NAME} broken on macOS (<${env.BUILD_URL}|Open>)" + + throw err + } + } +}, 'windows': { + node('windows') { + try { + checkout scm + + stage('Build Windows') { + bat '''CALL "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 +git clean -dfx +msbuild windows\\ZeroTierOne.sln +''' + } + } + catch (err) { + currentBuild.result = "FAILURE" + slackSend color: '#ff0000', message: "${env.JOB_NAME} broken on Windows (<${env.BUILD_URL}|Open>)" + + throw err + } + } +} + +slackSend color: "#00ff00", message: "${env.JOB_NAME} #${env.BUILD_NUMBER} Complete (<${env.BUILD_URL}|Show More...>)" diff --git a/zto/LICENSE.txt b/zto/LICENSE.txt new file mode 100644 index 0000000..8fd274b --- /dev/null +++ b/zto/LICENSE.txt @@ -0,0 +1,37 @@ +ZeroTier One - Network Virtualization Everywhere +Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +See LICENSE.GPL-3 for the full GNU GPL v3 license. + +-- + +You can be released from the requirements of the license by purchasing +a commercial license. Buying such a license is mandatory as soon as you +develop commercial closed-source software that incorporates or links +directly against ZeroTier software without disclosing the source code +of your own application. + +-- + +The above license does not apply to third party code included with or +linked against by ZeroTier software. See the third party code section +of the AUTHORS.md for an index of third party software included in +this software repository. + +Licenses for third party code are all relatively permissive: MIT, +BSD, and public domain. The only exception is the tap-windows driver +which is under the GPLv2, but this is only needed to produce the +binary tap device driver used by the ZeroTier service on Windows. diff --git a/zto/RELEASE-NOTES.md b/zto/RELEASE-NOTES.md index b54b7ea..195e888 100644 --- a/zto/RELEASE-NOTES.md +++ b/zto/RELEASE-NOTES.md @@ -1,16 +1,22 @@ ZeroTier Release Notes ====== -# 2017-03-17 -- Version 1.2.2 +# 2017-04-20 -- Version 1.2.4 -Version 1.2.2 fixes a few bugs discovered after the 1.2.0 release. These are: + * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users. + * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. My Macbook Pro (Core i5 2.8ghz) now does almost 1.5GiB/sec Salsa20/12 per core and a Raspberry Pi got a 2X boost. 64-bit ARM support and Windows support will take some work but should not be too hard. + * Refactored code that manages credentials to greatly reduce memory use in most cases. This may also result in a small performance improvement. + * Reworked and simplified path selection and priority logic to fix path instability and dead path persistence edge cases. There have been some sporadic reports of persistent path instabilities and dead paths hanging around that take minutes to resolve. These have proven difficult to reproduce in house, but hopefully this will fix them. In any case it seems to speed up path establishment in our tests and it makes the code simpler and more readable. + * Eliminated some unused cruft from the code around path management and in the peer class. + * Fixed an issue causing build problems on some MIPS architecture systems. + * Fixed Windows forgetting routes on sleep/wake or in some other circumstances. (GitHub issue #465) + +# 2017-03-17 -- Version 1.2.2 * A bug causing unreliable multicast propagation (GitHub issue #461). * A crash in ARM binaries due to a build chain and flags problem. * A bug in the network controller preventing members from being listed (GitHub issue #460). ------- - # 2017-03-14 -- Version 1.2.0 Version 1.2.0 is a major milestone release representing almost nine months of work. It includes our rules engine for distributed network packet filtering and security monitoring, federated roots, and many other architectural and UI improvements and bug fixes. diff --git a/zto/controller/EmbeddedNetworkController.cpp b/zto/controller/EmbeddedNetworkController.cpp index ce56e90..8ff8eb8 100644 --- a/zto/controller/EmbeddedNetworkController.cpp +++ b/zto/controller/EmbeddedNetworkController.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2015 ZeroTier, Inc. + * Copyright (C) 2011-2015 ZeroTier, Inc-> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,8 +30,9 @@ #include #include #include -#include #include +#include +#include #include "../include/ZeroTierOne.h" #include "../node/Constants.hpp" @@ -58,9 +59,6 @@ using json = nlohmann::json; // Min duration between requests for an address/nwid combo to prevent floods #define ZT_NETCONF_MIN_REQUEST_PERIOD 1000 -// Nodes are considered active if they've queried in less than this long -#define ZT_NETCONF_NODE_ACTIVE_THRESHOLD (ZT_NETWORK_AUTOCONF_DELAY * 2) - namespace ZeroTier { static json _renderRule(ZT_VirtualNetworkRule &rule) @@ -430,22 +428,24 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule) EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath) : _startTime(OSUtils::now()), - _threadsStarted(false), + _running(true), _db(dbPath), _node(node) { - OSUtils::mkdir(dbPath); - OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions } EmbeddedNetworkController::~EmbeddedNetworkController() { - Mutex::Lock _l(_threads_m); - if (_threadsStarted) { - for(int i=0;i<(ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT*2);++i) - _queue.post((_RQEntry *)0); - for(int i=0;i t; + { + Mutex::Lock _l(_threads_m); + _running = false; + t = _threads; + } + if (t.size() > 0) { + _queue.stop(); + for(std::vector::iterator i(t.begin());i!=t.end();++i) + Thread::join(*i); } } @@ -464,22 +464,14 @@ void EmbeddedNetworkController::request( { if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; - - { - Mutex::Lock _l(_threads_m); - if (!_threadsStarted) { - for(int i=0;inwid = nwid; qe->requestPacketId = requestPacketId; qe->fromAddr = fromAddr; qe->identity = identity; qe->metaData = metaData; + qe->type = _RQEntry::RQENTRY_TYPE_REQUEST; _queue.post(qe); } @@ -499,11 +491,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); json network; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids); - } - if (!network.size()) + if (!_db.getNetwork(nwid,network)) return 404; if (path.size() >= 3) { @@ -512,48 +500,35 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( if (path.size() >= 4) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString()); - } - if (!member.size()) + if (!_db.getNetworkMember(nwid,address,member)) return 404; - - _addMemberNonPersistedFields(member,OSUtils::now()); + _addMemberNonPersistedFields(nwid,address,member,OSUtils::now()); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; - - return 200; } else { - - Mutex::Lock _l(_db_m); - responseBody = "{"; - _db.filter((std::string("network/") + nwids + "/member/"),[&responseBody](const std::string &n,const json &member) { + _db.eachMember(nwid,[&responseBody](uint64_t networkId,uint64_t nodeId,const json &member) { if ((member.is_object())&&(member.size() > 0)) { responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(OSUtils::jsonString(member["id"],"0")); responseBody.append("\":"); responseBody.append(OSUtils::jsonString(member["revision"],"0")); } - return true; // never delete }); responseBody.push_back('}'); responseContentType = "application/json"; - - return 200; } + return 200; } // else 404 } else { const uint64_t now = OSUtils::now(); - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - _addNetworkNonPersistedFields(network,now,nmi); + JSONDB::NetworkSummaryInfo ns; + _db.getNetworkSummaryInfo(nwid,ns); + _addNetworkNonPersistedFields(network,now,ns); responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; @@ -561,24 +536,20 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } } else if (path.size() == 1) { - std::set networkIds; - { - Mutex::Lock _l(_db_m); - _db.filter("network/",[&networkIds](const std::string &n,const json &obj) { - if (n.length() == (16 + 8)) - networkIds.insert(n.substr(8)); - return true; // do not delete - }); - } + std::vector networkIds(_db.networkIds()); + std::sort(networkIds.begin(),networkIds.end()); + char tmp[64]; responseBody.push_back('['); - for(std::set::iterator i(networkIds.begin());i!=networkIds.end();++i) { - responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); - responseBody.append(*i); - responseBody.append("\""); + for(std::vector::const_iterator i(networkIds.begin());i!=networkIds.end();++i) { + if (responseBody.length() > 1) + responseBody.push_back(','); + Utils::snprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i); + responseBody.append(tmp); } responseBody.push_back(']'); responseContentType = "application/json"; + return 200; } // else 404 @@ -637,10 +608,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member; - { - Mutex::Lock _l(_db_m); - member = _db.get("network",nwids,"member",Address(address).toString()); - } + _db.getNetworkMember(nwid,address,member); json origMember(member); // for detecting changes _initMember(member); @@ -664,13 +632,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( // Member is being de-authorized, so spray Revocation objects to all online members if (!newAuth) { - _clearNetworkMemberInfoCache(nwid); - Revocation rev(_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); + Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); - Mutex::Lock _l(_lastRequestTime_m); - for(std::map< std::pair,uint64_t >::iterator i(_lastRequestTime.begin());i!=_lastRequestTime.end();++i) { - if ((now - i->second) < ZT_NETWORK_AUTOCONF_DELAY) - _node->ncSendRevocation(Address(i->first.first),rev); + + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))) + _node->ncSendRevocation(Address(i->first.nodeId),rev); } } } @@ -733,22 +701,25 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["address"] = addrs; // legacy member["nwid"] = nwids; + _removeMemberNonPersistedFields(member); if (member != origMember) { - member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",Address(address).toString(),member); - } - _pushMemberUpdate(now,nwid,member); + _db.saveNetworkMember(nwid,address,member); + + // Push update to member if online + try { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,address)]; + if ((ms.online(now))&&(ms.lastRequestMetaData)) + request(nwid,InetAddress(),0,ms.identity,ms.lastRequestMetaData); + } catch ( ... ) {} } - // Add non-persisted fields - member["clock"] = now; - + _addMemberNonPersistedFields(nwid,address,member,now); responseBody = OSUtils::jsonDump(member); responseContentType = "application/json"; + return 200; } else if ((path.size() == 3)&&(path[2] == "test")) { @@ -808,31 +779,27 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } else { // POST to network ID - json network; - { - Mutex::Lock _l(_db_m); - - // Magic ID ending with ______ picks a random unused network ID - if (path[1].substr(10) == "______") { - nwid = 0; - uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; - uint64_t nwidPostfix = 0; - for(unsigned long k=0;k<100000;++k) { // sanity limit on trials - Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); - uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); - if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)tryNwid); - if (_db.get("network",nwids).size() <= 0) { - nwid = tryNwid; - break; - } + // Magic ID ending with ______ picks a random unused network ID + if (path[1].substr(10) == "______") { + nwid = 0; + uint64_t nwidPrefix = (Utils::hexStrToU64(path[1].substr(0,10).c_str()) << 24) & 0xffffffffff000000ULL; + uint64_t nwidPostfix = 0; + for(unsigned long k=0;k<100000;++k) { // sanity limit on trials + Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix)); + uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL); + if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL; + if (!_db.hasNetwork(tryNwid)) { + nwid = tryNwid; + break; } - if (!nwid) - return 503; } - - network = _db.get("network",nwids); + if (!nwid) + return 503; } + Utils::snprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid); + + json network; + _db.getNetwork(nwid,network); json origNetwork(network); // for detecting changes _initNetwork(network); @@ -1018,9 +985,10 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( json ntag = json::object(); const uint64_t tagId = OSUtils::jsonInt(tag["id"],0ULL); ntag["id"] = tagId; - if (tag.find("default") == tag.end()) - ntag["default"] = json(); - else ntag["default"] = OSUtils::jsonInt(tag["default"],0ULL); + json &dfl = tag["default"]; + if (dfl.is_null()) + ntag["default"] = dfl; + else ntag["default"] = OSUtils::jsonInt(dfl,0ULL); ntags[tagId] = ntag; } } @@ -1041,25 +1009,23 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy + _removeNetworkNonPersistedFields(network); if (network != origNetwork) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); - network["lastModified"] = now; - { - Mutex::Lock _l(_db_m); - _db.put("network",nwids,network); - } + _db.saveNetwork(nwid,network); - // Send an update to all members of the network - _db.filter((std::string("network/") + nwids + "/member/"),[this,&now,&nwid](const std::string &n,const json &obj) { - _pushMemberUpdate(now,nwid,obj); - return true; // do not delete - }); + // Send an update to all members of the network that are online + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) { + if ((i->first.networkId == nwid)&&(i->second.online(now))&&(i->second.lastRequestMetaData)) + request(nwid,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData); + } } - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - _addNetworkNonPersistedFields(network,now,nmi); + JSONDB::NetworkSummaryInfo ns; + _db.getNetworkSummaryInfo(nwid,ns); + _addNetworkNonPersistedFields(network,now,ns); responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; @@ -1068,13 +1034,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } // else 404 - } else if (path[0] == "dbtest") { + } else if (path[0] == "ping") { - json testRec; - const uint64_t now = OSUtils::now(); - testRec["clock"] = now; - testRec["uptime"] = (now - _startTime); - _db.put("dbtest",testRec); + _startThreads(); + _RQEntry *qe = new _RQEntry; + qe->type = _RQEntry::RQENTRY_TYPE_PING; + _queue.post(qe); + + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"{\"clock\":%llu,\"ping\":%s}",(unsigned long long)now,OSUtils::jsonDump(b).c_str()); + responseBody = tmp; + responseContentType = "application/json"; + + return 200; } @@ -1095,25 +1067,16 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( if (path[0] == "network") { if ((path.size() >= 2)&&(path[1].length() == 16)) { const uint64_t nwid = Utils::hexStrToU64(path[1].c_str()); - - char nwids[24]; - Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids); - } - if (!network.size()) - return 404; - if (path.size() >= 3) { if ((path.size() == 4)&&(path[2] == "member")&&(path[3].length() == 10)) { const uint64_t address = Utils::hexStrToU64(path[3].c_str()); - Mutex::Lock _l(_db_m); + json member = _db.eraseNetworkMember(nwid,address); - json member = _db.get("network",nwids,"member",Address(address).toString()); - _db.erase("network",nwids,"member",Address(address).toString()); + { + Mutex::Lock _l(_memberStatus_m); + _memberStatus.erase(_MemberStatusKey(nwid,address)); + } if (!member.size()) return 404; @@ -1122,16 +1085,19 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE( return 200; } } else { - Mutex::Lock _l(_db_m); + json network = _db.eraseNetwork(nwid); - std::string pfx("network/"); pfx.append(nwids); - _db.filter(pfx,[](const std::string &n,const json &obj) { - return false; // delete - }); - - Mutex::Lock _l2(_nmiCache_m); - _nmiCache.erase(nwid); + { + Mutex::Lock _l(_memberStatus_m); + for(auto i=_memberStatus.begin();i!=_memberStatus.end();) { + if (i->first.networkId == nwid) + _memberStatus.erase(i++); + else ++i; + } + } + if (!network.size()) + return 404; responseBody = OSUtils::jsonDump(network); responseContentType = "application/json"; return 200; @@ -1147,23 +1113,52 @@ void EmbeddedNetworkController::threadMain() throw() { uint64_t lastCircuitTestCheck = 0; - for(;;) { - _RQEntry *const qe = _queue.get(); // waits on next request - if (!qe) break; // enqueue a NULL to terminate threads + _RQEntry *qe = (_RQEntry *)0; + while ((_running)&&(_queue.get(qe))) { try { - _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + if (qe->type == _RQEntry::RQENTRY_TYPE_REQUEST) { + _request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData); + } else if (qe->type == _RQEntry::RQENTRY_TYPE_PING) { + const uint64_t now = OSUtils::now(); + bool first = true; + std::string pong("{\"memberStatus\":{"); + { + Mutex::Lock _l(_memberStatus_m); + pong.reserve(64 * _memberStatus.size()); + _db.eachId([this,&pong,&now,&first](uint64_t networkId,uint64_t nodeId) { + char tmp[64]; + uint64_t lrt = 0ULL; + auto ms = this->_memberStatus.find(_MemberStatusKey(networkId,nodeId)); + if (ms != _memberStatus.end()) + lrt = ms->second.lastRequestTime; + Utils::snprintf(tmp,sizeof(tmp),"%s\"%.16llx-%.10llx\":%llu", + (first) ? "" : ",", + (unsigned long long)networkId, + (unsigned long long)nodeId, + (unsigned long long)lrt); + pong.append(tmp); + first = false; + }); + } + char tmp2[256]; + Utils::snprintf(tmp2,sizeof(tmp2),"},\"clock\":%llu,\"startTime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime); + pong.append(tmp2); + _db.writeRaw("pong",pong); + } } catch ( ... ) {} delete qe; - uint64_t now = OSUtils::now(); - if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - lastCircuitTestCheck = now; - Mutex::Lock _l(_tests_m); - for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { - if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { - _node->circuitTestEnd(&(*i)); - _tests.erase(i++); - } else ++i; + if (_running) { + uint64_t now = OSUtils::now(); + if ((now - lastCircuitTestCheck) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + lastCircuitTestCheck = now; + Mutex::Lock _l(_tests_m); + for(std::list< ZT_CircuitTest >::iterator i(_tests.begin());i!=_tests.end();) { + if ((now - i->timestamp) > ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION) { + _node->circuitTestEnd(&(*i)); + _tests.erase(i++); + } else ++i; + } } } } @@ -1171,7 +1166,7 @@ void EmbeddedNetworkController::threadMain() void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report) { - char tmp[1024],id[128]; + char tmp[2048],id[128]; EmbeddedNetworkController *const self = reinterpret_cast(test->ptr); if ((!test)||(!report)||(!test->credentialNetworkId)) return; // sanity check @@ -1180,6 +1175,7 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes Utils::snprintf(id,sizeof(id),"network/%.16llx/test/%.16llx-%.16llx-%.10llx-%.10llx",test->credentialNetworkId,test->testId,now,report->upstream,report->current); Utils::snprintf(tmp,sizeof(tmp), "{\"id\": \"%s\"," + "\"objtype\": \"circuit_test\"," "\"timestamp\": %llu," "\"networkId\": \"%.16llx\"," "\"testId\": \"%.16llx\"," @@ -1222,7 +1218,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes reinterpret_cast(&(report->receivedFromRemoteAddress))->toString().c_str(), ((double)report->receivedFromLinkQuality / (double)ZT_PATH_LINK_QUALITY_MAX)); - Mutex::Lock _l(self->_db_m); self->_db.writeRaw(id,std::string(tmp)); } @@ -1233,37 +1228,30 @@ void EmbeddedNetworkController::_request( const Identity &identity, const Dictionary &metaData) { + char nwids[24]; + JSONDB::NetworkSummaryInfo ns; + json network,member,origMember; + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; const uint64_t now = OSUtils::now(); if (requestPacketId) { - Mutex::Lock _l(_lastRequestTime_m); - uint64_t &lrt = _lastRequestTime[std::pair(identity.address().toInt(),nwid)]; - if ((now - lrt) <= ZT_NETCONF_MIN_REQUEST_PERIOD) + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + if ((now - ms.lastRequestTime) <= ZT_NETCONF_MIN_REQUEST_PERIOD) return; - lrt = now; + ms.lastRequestTime = now; } - char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network; - json member; - { - Mutex::Lock _l(_db_m); - network = _db.get("network",nwids); - member = _db.get("network",nwids,"member",identity.address().toString()); - } - - if (!network.size()) { + if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; } - - const bool newMember = (member.size() == 0); - - json origMember(member); // for detecting modification later + origMember = member; + const bool newMember = ((!member.is_object())||(member.size() == 0)); _initMember(member); { @@ -1288,9 +1276,12 @@ void EmbeddedNetworkController::_request( } // These are always the same, but make sure they are set - member["id"] = identity.address().toString(); - member["address"] = member["id"]; - member["nwid"] = nwids; + { + const std::string addrs(identity.address().toString()); + member["id"] = addrs; + member["address"] = addrs; + member["nwid"] = nwids; + } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -1366,42 +1357,41 @@ void EmbeddedNetworkController::_request( member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); } - // Log this request - if (requestPacketId) { // only log if this is a request, not for generated pushes - json rlEntry = json::object(); - rlEntry["ts"] = now; - rlEntry["auth"] = (authorizedBy) ? true : false; - rlEntry["authBy"] = (authorizedBy) ? authorizedBy : ""; - rlEntry["vMajor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); - rlEntry["vMinor"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); - rlEntry["vRev"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); - rlEntry["vProto"] = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - if (fromAddr) - rlEntry["fromAddr"] = fromAddr.toString(); + if (authorizedBy) { + // Update version info and meta-data if authorized and if this is a genuine request + if (requestPacketId) { + const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0); + const uint64_t vMinor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,0); + const uint64_t vRev = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,0); + const uint64_t vProto = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,0); - json recentLog = json::array(); - recentLog.push_back(rlEntry); - json &oldLog = member["recentLog"]; - if (oldLog.is_array()) { - for(unsigned long i=0;i= ZT_NETCONF_DB_MEMBER_HISTORY_LENGTH) - break; + member["vMajor"] = vMajor; + member["vMinor"] = vMinor; + member["vRev"] = vRev; + member["vProto"] = vProto; + + { + Mutex::Lock _l(_memberStatus_m); + _MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,identity.address().toInt())]; + + ms.vMajor = (int)vMajor; + ms.vMinor = (int)vMinor; + ms.vRev = (int)vRev; + ms.vProto = (int)vProto; + ms.lastRequestMetaData = metaData; + ms.identity = identity; + + if (fromAddr) + ms.physicalAddr = fromAddr; + if (ms.physicalAddr) + member["physicalAddr"] = ms.physicalAddr.toString(); } } - member["recentLog"] = recentLog; - - // Also only do this on real requests - member["lastRequestMetaData"] = metaData.data(); - } - - // If they are not authorized, STOP! - if (!authorizedBy) { - if (origMember != member) { - member["lastModified"] = now; - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - } + } else { + // If they are not authorized, STOP! + _removeMemberNonPersistedFields(member); + if (origMember != member) + _db.saveNetworkMember(nwid,identity.address().toInt(),member); _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); return; } @@ -1410,16 +1400,12 @@ void EmbeddedNetworkController::_request( // If we made it this far, they are authorized. // ------------------------------------------------------------------------- - NetworkConfig nc; - _NetworkMemberInfo nmi; - _getNetworkMemberInfo(now,nwid,nmi); - uint64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA; - if (now > nmi.mostRecentDeauthTime) { + if (now > ns.mostRecentDeauthTime) { // If we recently de-authorized a member, shrink credential TTL/max delta to // be below the threshold required to exclude it. Cap this to a min/max to // prevent jitter or absurdly large values. - const uint64_t deauthWindow = now - nmi.mostRecentDeauthTime; + const uint64_t deauthWindow = now - ns.mostRecentDeauthTime; if (deauthWindow < ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA) { credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA; } else if (deauthWindow < (ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA + 5000ULL)) { @@ -1427,20 +1413,21 @@ void EmbeddedNetworkController::_request( } } - nc.networkId = nwid; - nc.type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; - nc.timestamp = now; - nc.credentialTimeMaxDelta = credentialtmd; - nc.revision = OSUtils::jsonInt(network["revision"],0ULL); - nc.issuedTo = identity.address(); - if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; - if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc.flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; - Utils::scopy(nc.name,sizeof(nc.name),OSUtils::jsonString(network["name"],"").c_str()); - nc.multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); + std::auto_ptr nc(new NetworkConfig()); - for(std::set
::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) { - nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); - } + nc->networkId = nwid; + nc->type = OSUtils::jsonBool(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; + nc->timestamp = now; + nc->credentialTimeMaxDelta = credentialtmd; + nc->revision = OSUtils::jsonInt(network["revision"],0ULL); + nc->issuedTo = identity.address(); + if (OSUtils::jsonBool(network["enableBroadcast"],true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; + if (OSUtils::jsonBool(network["allowPassiveBridging"],false)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ALLOW_PASSIVE_BRIDGING; + Utils::scopy(nc->name,sizeof(nc->name),OSUtils::jsonString(network["name"],"").c_str()); + nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL); + + for(std::vector
::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab) + nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); json &v4AssignMode = network["v4AssignMode"]; json &v6AssignMode = network["v6AssignMode"]; @@ -1456,15 +1443,15 @@ void EmbeddedNetworkController::_request( // Old versions with no rules engine support get an allow everything rule. // Since rules are enforced bidirectionally, newer versions *will* still // enforce rules on the inbound side. - nc.ruleCount = 1; - nc.rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; + nc->ruleCount = 1; + nc->rules[0].t = ZT_NETWORK_RULE_ACTION_ACCEPT; } else { if (rules.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) + if (nc->ruleCount >= ZT_MAX_NETWORK_RULES) break; - if (_parseRule(rules[i],nc.rules[nc.ruleCount])) - ++nc.ruleCount; + if (_parseRule(rules[i],nc->rules[nc->ruleCount])) + ++nc->ruleCount; } } @@ -1508,10 +1495,10 @@ void EmbeddedNetworkController::_request( ++caprc; } } - nc.capabilities[nc.capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); - if (nc.capabilities[nc.capabilityCount].sign(_signingId,identity.address())) - ++nc.capabilityCount; - if (nc.capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) + nc->capabilities[nc->capabilityCount] = Capability((uint32_t)capId,nwid,now,1,capr,caprc); + if (nc->capabilities[nc->capabilityCount].sign(_signingId,identity.address())) + ++nc->capabilityCount; + if (nc->capabilityCount >= ZT_MAX_NETWORK_CAPABILITIES) break; } } @@ -1542,17 +1529,17 @@ void EmbeddedNetworkController::_request( } } for(std::map< uint32_t,uint32_t >::const_iterator t(memberTagsById.begin());t!=memberTagsById.end();++t) { - if (nc.tagCount >= ZT_MAX_NETWORK_TAGS) + if (nc->tagCount >= ZT_MAX_NETWORK_TAGS) break; - nc.tags[nc.tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); - if (nc.tags[nc.tagCount].sign(_signingId)) - ++nc.tagCount; + nc->tags[nc->tagCount] = Tag(nwid,now,identity.address(),t->first,t->second); + if (nc->tags[nc->tagCount].sign(_signingId)) + ++nc->tagCount; } } if (routes.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_ROUTES) + if (nc->routeCount >= ZT_MAX_NETWORK_ROUTES) break; json &route = routes[i]; json &target = route["target"]; @@ -1562,11 +1549,11 @@ void EmbeddedNetworkController::_request( InetAddress v; if (via.is_string()) v.fromString(via.get()); if ((t.ss_family == AF_INET)||(t.ss_family == AF_INET6)) { - ZT_VirtualNetworkRoute *r = &(nc.routes[nc.routeCount]); + ZT_VirtualNetworkRoute *r = &(nc->routes[nc->routeCount]); *(reinterpret_cast(&(r->target))) = t; if (v.ss_family == t.ss_family) *(reinterpret_cast(&(r->via))) = v; - ++nc.routeCount; + ++nc->routeCount; } } } @@ -1575,13 +1562,13 @@ void EmbeddedNetworkController::_request( const bool noAutoAssignIps = OSUtils::jsonBool(member["noAutoAssignIps"],false); if ((v6AssignMode.is_object())&&(!noAutoAssignIps)) { - if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["rfc4193"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } - if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + if ((OSUtils::jsonBool(v6AssignMode["6plane"],false))&&(nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc->staticIps[nc->staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; } } @@ -1599,15 +1586,15 @@ void EmbeddedNetworkController::_request( // this route, ignoring the netmask bits field of the assigned IP itself. Using that was worthless and a source // of user error / poor UX. int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } if (routedNetmaskBits > 0) { - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { ip.setPort(routedNetmaskBits); - nc.staticIps[nc.staticIpCount++] = ip; + nc->staticIps[nc->staticIpCount++] = ip; } if (ip.ss_family == AF_INET) haveManagedIpv4AutoAssignment = true; @@ -1658,20 +1645,19 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = 0; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->containsAddress(ip6)) ) - routedNetmaskBits = reinterpret_cast(&(nc.routes[rk].target))->netmaskBits(); + for(unsigned int rk=0;rkrouteCount;++rk) { + if ( (!nc->routes[rk].via.ss_family) && (nc->routes[rk].target.ss_family == AF_INET6) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip6)) ) + routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { + if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip6)) ) { ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) - nc.staticIps[nc.staticIpCount++] = ip6; + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) + nc->staticIps[nc->staticIpCount++] = ip6; haveManagedIpv6AutoAssignment = true; - _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } @@ -1704,10 +1690,10 @@ void EmbeddedNetworkController::_request( // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = -1; - for(unsigned int rk=0;rk(&(nc.routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc.routes[rk].target))->sin_port)); + for(unsigned int rk=0;rkrouteCount;++rk) { + if (nc->routes[rk].target.ss_family == AF_INET) { + uint32_t targetIp = Utils::ntoh((uint32_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_addr.s_addr)); + int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { routedNetmaskBits = targetBits; break; @@ -1717,17 +1703,16 @@ void EmbeddedNetworkController::_request( // If it's routed, then try to claim and assign it and if successful end loop const InetAddress ip4(Utils::hton(ip),0); - if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { + if ( (routedNetmaskBits > 0) && (!std::binary_search(ns.allocatedIps.begin(),ns.allocatedIps.end(),ip4)) ) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; - if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in *const v4ip = reinterpret_cast(&(nc.staticIps[nc.staticIpCount++])); + if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { + struct sockaddr_in *const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); v4ip->sin_family = AF_INET; v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); v4ip->sin_addr.s_addr = Utils::hton(ip); } haveManagedIpv4AutoAssignment = true; - _clearNetworkMemberInfoCache(nwid); // clear cache to prevent IP assignment duplication on many rapid assigns break; } } @@ -1737,114 +1722,27 @@ void EmbeddedNetworkController::_request( } // Issue a certificate of ownership for all static IPs - if (nc.staticIpCount) { - nc.certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); - for(unsigned int i=0;istaticIpCount) { + nc->certificatesOfOwnership[0] = CertificateOfOwnership(nwid,now,identity.address(),1); + for(unsigned int i=0;istaticIpCount;++i) + nc->certificatesOfOwnership[0].addThing(nc->staticIps[i]); + nc->certificatesOfOwnership[0].sign(_signingId); + nc->certificateOfOwnershipCount = 1; } CertificateOfMembership com(now,credentialtmd,nwid,identity.address()); if (com.sign(_signingId)) { - nc.com = com; + nc->com = com; } else { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR); return; } - if (member != origMember) { - member["lastModified"] = now; - Mutex::Lock _l(_db_m); - _db.put("network",nwids,"member",identity.address().toString(),member); - } + _removeMemberNonPersistedFields(member); + if (member != origMember) + _db.saveNetworkMember(nwid,identity.address().toInt(),member); - _sender->ncSendConfig(nwid,requestPacketId,identity.address(),nc,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); -} - -void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) -{ - char pfx[256]; - Utils::snprintf(pfx,sizeof(pfx),"network/%.16llx/member",nwid); - - { - Mutex::Lock _l(_nmiCache_m); - std::map::iterator c(_nmiCache.find(nwid)); - if ((c != _nmiCache.end())&&((now - c->second.nmiTimestamp) < 1000)) { // a short duration cache but limits CPU use on big networks - nmi = c->second; - return; - } - } - - { - Mutex::Lock _l(_db_m); - _db.filter(pfx,[&nmi,&now](const std::string &n,const json &member) { - try { - if (OSUtils::jsonBool(member["authorized"],false)) { - ++nmi.authorizedMemberCount; - - if (member.count("recentLog")) { - const json &mlog = member["recentLog"]; - if ((mlog.is_array())&&(mlog.size() > 0)) { - const json &mlog1 = mlog[0]; - if (mlog1.is_object()) { - if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) - ++nmi.activeMemberCount; - } - } - } - - if (OSUtils::jsonBool(member["activeBridge"],false)) { - nmi.activeBridges.insert(Address(Utils::hexStrToU64(OSUtils::jsonString(member["id"],"0000000000").c_str()))); - } - - if (member.count("ipAssignments")) { - const json &mips = member["ipAssignments"]; - if (mips.is_array()) { - for(unsigned long i=0;i 0)&&(mdstr.length() > 0)) { - const Identity id(idstr); - bool online; - { - Mutex::Lock _l(_lastRequestTime_m); - std::map< std::pair,uint64_t >::iterator lrt(_lastRequestTime.find(std::pair(id.address().toInt(),nwid))); - online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) ); - } - if (online) { - Dictionary *metaData = new Dictionary(mdstr.c_str()); - try { - this->request(nwid,InetAddress(),0,id,*metaData); - } catch ( ... ) {} - delete metaData; - } - } - } catch ( ... ) {} + _sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6); } } // namespace ZeroTier diff --git a/zto/controller/EmbeddedNetworkController.hpp b/zto/controller/EmbeddedNetworkController.hpp index 0ae2f3b..ade7eb2 100644 --- a/zto/controller/EmbeddedNetworkController.hpp +++ b/zto/controller/EmbeddedNetworkController.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include "../node/Constants.hpp" @@ -34,6 +35,7 @@ #include "../node/Utils.hpp" #include "../node/Address.hpp" #include "../node/InetAddress.hpp" +#include "../node/NonCopyable.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/Thread.hpp" @@ -43,9 +45,6 @@ #include "JSONDB.hpp" -// Number of background threads to start -- not actually started until needed -#define ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT 4 - // TTL for circuit tests #define ZT_EMBEDDEDNETWORKCONTROLLER_CIRCUIT_TEST_EXPIRATION 120000 @@ -53,7 +52,7 @@ namespace ZeroTier { class Node; -class EmbeddedNetworkController : public NetworkController +class EmbeddedNetworkController : public NetworkController,NonCopyable { public: /** @@ -105,26 +104,28 @@ private: InetAddress fromAddr; Identity identity; Dictionary metaData; - }; - - // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places - struct _NetworkMemberInfo - { - _NetworkMemberInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} - std::set
activeBridges; - std::set allocatedIps; - unsigned long authorizedMemberCount; - unsigned long activeMemberCount; - unsigned long totalMemberCount; - uint64_t mostRecentDeauthTime; - uint64_t nmiTimestamp; // time this NMI structure was computed + enum { + RQENTRY_TYPE_REQUEST = 0, + RQENTRY_TYPE_PING = 1 + } type; }; static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report); void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary &metaData); - void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); - inline void _clearNetworkMemberInfoCache(const uint64_t nwid) { Mutex::Lock _l(_nmiCache_m); _nmiCache.erase(nwid); } - void _pushMemberUpdate(uint64_t now,uint64_t nwid,const nlohmann::json &member); + + inline void _startThreads() + { + Mutex::Lock _l(_threads_m); + if (_threads.size() == 0) { + long hwc = (long)std::thread::hardware_concurrency(); + if (hwc < 1) + hwc = 1; + else if (hwc > 16) + hwc = 16; + for(long i=0;i _queue; - Thread _threads[ZT_EMBEDDEDNETWORKCONTROLLER_BACKGROUND_THREAD_COUNT]; - bool _threadsStarted; + std::vector _threads; Mutex _threads_m; - std::map _nmiCache; - Mutex _nmiCache_m; - JSONDB _db; - Mutex _db_m; Node *const _node; std::string _path; @@ -201,8 +221,33 @@ private: std::list< ZT_CircuitTest > _tests; Mutex _tests_m; - std::map< std::pair,uint64_t > _lastRequestTime; // last request time by - Mutex _lastRequestTime_m; + struct _MemberStatusKey + { + _MemberStatusKey() : networkId(0),nodeId(0) {} + _MemberStatusKey(const uint64_t nwid,const uint64_t nid) : networkId(nwid),nodeId(nid) {} + uint64_t networkId; + uint64_t nodeId; + inline bool operator==(const _MemberStatusKey &k) const { return ((k.networkId == networkId)&&(k.nodeId == nodeId)); } + }; + struct _MemberStatus + { + _MemberStatus() : lastRequestTime(0),vMajor(-1),vMinor(-1),vRev(-1),vProto(-1) {} + uint64_t lastRequestTime; + int vMajor,vMinor,vRev,vProto; + Dictionary lastRequestMetaData; + Identity identity; + InetAddress physicalAddr; // last known physical address + inline bool online(const uint64_t now) const { return ((now - lastRequestTime) < (ZT_NETWORK_AUTOCONF_DELAY * 2)); } + }; + struct _MemberStatusHash + { + inline std::size_t operator()(const _MemberStatusKey &networkIdNodeId) const + { + return (std::size_t)(networkIdNodeId.networkId + networkIdNodeId.nodeId); + } + }; + std::unordered_map< _MemberStatusKey,_MemberStatus,_MemberStatusHash > _memberStatus; + Mutex _memberStatus_m; }; } // namespace ZeroTier diff --git a/zto/controller/JSONDB.cpp b/zto/controller/JSONDB.cpp index 8b6de9b..31c0cc0 100644 --- a/zto/controller/JSONDB.cpp +++ b/zto/controller/JSONDB.cpp @@ -18,126 +18,436 @@ #include "JSONDB.hpp" +#define ZT_JSONDB_HTTP_TIMEOUT 60000 + namespace ZeroTier { static const nlohmann::json _EMPTY_JSON(nlohmann::json::object()); +static const std::map _ZT_JSONDB_GET_HEADERS; + +JSONDB::JSONDB(const std::string &basePath) : + _basePath(basePath), + _summaryThreadRun(true) +{ + if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) { + // TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported. + // Typically it's just used with 127.0.0.1 anyway. + std::string hn = _basePath.substr(7); + std::size_t hnend = hn.find_first_of('/'); + if (hnend != std::string::npos) + hn = hn.substr(0,hnend); + std::size_t hnsep = hn.find_last_of(':'); + if (hnsep != std::string::npos) + hn[hnsep] = '/'; + _httpAddr.fromString(hn); + if (hnend != std::string::npos) + _basePath = _basePath.substr(7 + hnend); + if (_basePath.length() == 0) + _basePath = "/"; + if (_basePath[0] != '/') + _basePath = std::string("/") + _basePath; + } else { + OSUtils::mkdir(_basePath.c_str()); + OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions + } + + unsigned int cnt = 0; + while (!_load(_basePath)) { + if ((++cnt & 7) == 0) + fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str()); + Thread::sleep(250); + } + + for(std::unordered_map::iterator n(_networks.begin());n!=_networks.end();++n) + _recomputeSummaryInfo(n->first); + for(;;) { + _summaryThread_m.lock(); + if (_summaryThreadToDo.empty()) { + _summaryThread_m.unlock(); + break; + } + _summaryThread_m.unlock(); + Thread::sleep(50); + } +} + +JSONDB::~JSONDB() +{ + { + Mutex::Lock _l(_networks_m); + _networks.clear(); + } + Thread t; + { + Mutex::Lock _l(_summaryThread_m); + _summaryThreadRun = false; + t = _summaryThread; + } + if (t) + Thread::join(t); +} bool JSONDB::writeRaw(const std::string &n,const std::string &obj) { - if (!_isValidObjectName(n)) - return false; - - const std::string path(_genPath(n,true)); - if (!path.length()) - return false; - - const std::string buf(obj); - if (!OSUtils::writeFile(path.c_str(),buf)) - return false; - - return true; -} - -bool JSONDB::put(const std::string &n,const nlohmann::json &obj) -{ - if (!_isValidObjectName(n)) - return false; - - const std::string path(_genPath(n,true)); - if (!path.length()) - return false; - - const std::string buf(OSUtils::jsonDump(obj)); - if (!OSUtils::writeFile(path.c_str(),buf)) - return false; - - _E &e = _db[n]; - e.obj = obj; - - return true; -} - -const nlohmann::json &JSONDB::get(const std::string &n) -{ - if (!_isValidObjectName(n)) - return _EMPTY_JSON; - - std::map::iterator e(_db.find(n)); - if (e != _db.end()) - return e->second.obj; - - const std::string path(_genPath(n,false)); - if (!path.length()) - return _EMPTY_JSON; - std::string buf; - if (!OSUtils::readFile(path.c_str(),buf)) - return _EMPTY_JSON; - - _E &e2 = _db[n]; - try { - e2.obj = OSUtils::jsonParse(buf); - } catch ( ... ) { - e2.obj = _EMPTY_JSON; - buf = "{}"; - } - - return e2.obj; -} - -void JSONDB::erase(const std::string &n) -{ - if (!_isValidObjectName(n)) - return; - - std::string path(_genPath(n,true)); - if (!path.length()) - return; - - OSUtils::rm(path.c_str()); - _db.erase(n); -} - -void JSONDB::_reload(const std::string &p,const std::string &b) -{ - std::vector dl(OSUtils::listDirectory(p.c_str())); - for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { - if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { - this->get(b + di->substr(0,di->length() - 5)); - } else { - this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR)); - } - } -} - -bool JSONDB::_isValidObjectName(const std::string &n) -{ - if (n.length() == 0) - return false; - const char *p = n.c_str(); - char c; - // For security reasons we should not allow dots, backslashes, or other path characters or potential path characters. - while ((c = *(p++))) { - if (!( ((c >= 'a')&&(c <= 'z')) || ((c >= 'A')&&(c <= 'Z')) || ((c >= '0')&&(c <= '9')) || (c == '/') || (c == '_') || (c == '~') || (c == '-') )) + if (_httpAddr) { + std::map headers; + std::string body; + std::map reqHeaders; + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length()); + reqHeaders["Content-Length"] = tmp; + reqHeaders["Content-Type"] = "application/json"; + const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body); + return (sc == 200); + } else { + const std::string path(_genPath(n,true)); + if (!path.length()) return false; + return OSUtils::writeFile(path.c_str(),obj); } +} + +bool JSONDB::hasNetwork(const uint64_t networkId) const +{ + Mutex::Lock _l(_networks_m); + return (_networks.find(networkId) != _networks.end()); +} + +bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const +{ + Mutex::Lock _l(_networks_m); + const std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + config = nlohmann::json::from_msgpack(i->second.config); return true; } +bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + const std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + ns = i->second.summaryInfo; + return true; +} + +int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const +{ + Mutex::Lock _l(_networks_m); + const std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return 0; + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return 1; + networkConfig = nlohmann::json::from_msgpack(i->second.config); + memberConfig = nlohmann::json::from_msgpack(j->second); + ns = i->second.summaryInfo; + return 3; +} + +bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const +{ + Mutex::Lock _l(_networks_m); + const std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return false; + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return false; + memberConfig = nlohmann::json::from_msgpack(j->second); + return true; +} + +void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig) +{ + char n[64]; + Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + writeRaw(n,OSUtils::jsonDump(networkConfig)); + { + Mutex::Lock _l(_networks_m); + _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig); + } + _recomputeSummaryInfo(networkId); +} + +void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig) +{ + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + writeRaw(n,OSUtils::jsonDump(memberConfig)); + { + Mutex::Lock _l(_networks_m); + _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); + } + _recomputeSummaryInfo(networkId); +} + +nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) +{ + if (!_httpAddr) { // Member deletion is done by Central in harnessed mode, and deleting the cache network entry also deletes all members + std::vector memberIds; + { + Mutex::Lock _l(_networks_m); + const std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; + for(std::unordered_map< uint64_t,std::vector >::iterator m(i->second.members.begin());m!=i->second.members.end();++m) + memberIds.push_back(m->first); + } + for(std::vector::iterator m(memberIds.begin());m!=memberIds.end();++m) + eraseNetworkMember(networkId,*m,false); + } + + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId); + + if (_httpAddr) { + // Deletion is currently done by Central in harnessed mode + //std::map headers; + //std::string body; + //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + } else { + const std::string path(_genPath(n,false)); + if (path.length()) + OSUtils::rm(path.c_str()); + } + + { + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; // sanity check, shouldn't happen + nlohmann::json tmp(nlohmann::json::from_msgpack(i->second.config)); + _networks.erase(i); + return tmp; + } +} + +nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo) +{ + char n[256]; + Utils::snprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId); + + if (_httpAddr) { + // Deletion is currently done by the caller in Central harnessed mode + //std::map headers; + //std::string body; + //Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + } else { + const std::string path(_genPath(n,false)); + if (path.length()) + OSUtils::rm(path.c_str()); + } + + { + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator i(_networks.find(networkId)); + if (i == _networks.end()) + return _EMPTY_JSON; + std::unordered_map< uint64_t,std::vector >::iterator j(i->second.members.find(nodeId)); + if (j == i->second.members.end()) + return _EMPTY_JSON; + nlohmann::json tmp(j->second); + i->second.members.erase(j); + if (recomputeSummaryInfo) + _recomputeSummaryInfo(networkId); + return tmp; + } +} + +void JSONDB::threadMain() + throw() +{ + std::vector todo; + while (_summaryThreadRun) { + Thread::sleep(10); + + { + Mutex::Lock _l(_summaryThread_m); + if (_summaryThreadToDo.empty()) + continue; + else _summaryThreadToDo.swap(todo); + } + + const uint64_t now = OSUtils::now(); + for(std::vector::iterator ii(todo.begin());ii!=todo.end();++ii) { + const uint64_t networkId = *ii; + + Mutex::Lock _l(_networks_m); + std::unordered_map::iterator n(_networks.find(networkId)); + if (n != _networks.end()) { + NetworkSummaryInfo &ns = n->second.summaryInfo; + ns.activeBridges.clear(); + ns.allocatedIps.clear(); + ns.authorizedMemberCount = 0; + ns.activeMemberCount = 0; + ns.totalMemberCount = 0; + ns.mostRecentDeauthTime = 0; + + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { + try { + nlohmann::json member(nlohmann::json::from_msgpack(m->second)); + + if (OSUtils::jsonBool(member["authorized"],false)) { + ++ns.authorizedMemberCount; + + try { + const nlohmann::json &mlog = member["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + const nlohmann::json &mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - OSUtils::jsonInt(mlog1["ts"],0ULL)) < (ZT_NETWORK_AUTOCONF_DELAY * 2)) + ++ns.activeMemberCount; + } + } + } catch ( ... ) {} + + try { + if (OSUtils::jsonBool(member["activeBridge"],false)) + ns.activeBridges.push_back(Address(m->first)); + } catch ( ... ) {} + + try { + const nlohmann::json &mips = member["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;isecond.summaryInfoLastComputed = now; + } + } + + todo.clear(); + } +} + +bool JSONDB::_load(const std::string &p) +{ + if (_httpAddr) { + // In HTTP harnessed mode we download our entire working data set on startup. + + std::string body; + std::map headers; + const unsigned int sc = Http::GET(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); + if (sc == 200) { + try { + nlohmann::json dbImg(OSUtils::jsonParse(body)); + std::string tmp; + if (dbImg.is_object()) { + Mutex::Lock _l(_networks_m); + for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) { + nlohmann::json &j = i.value(); + if (j.is_object()) { + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::hexStrToU64(id.c_str()); + if (nwid) + _networks[nwid].config = nlohmann::json::to_msgpack(j); + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::hexStrToU64(id.c_str()); + const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + } + } + } + return true; + } + } catch ( ... ) {} // invalid JSON, so maybe incomplete request + } + return false; + + } else { + // In regular mode we recursively read it from controller.d/ on disk + + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); + for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { + if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { + std::string buf; + if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) { + try { + nlohmann::json j(OSUtils::jsonParse(buf)); + std::string id(OSUtils::jsonString(j["id"],"0")); + std::string objtype(OSUtils::jsonString(j["objtype"],"")); + + if ((id.length() == 16)&&(objtype == "network")) { + const uint64_t nwid = Utils::strToU64(id.c_str()); + if (nwid) { + Mutex::Lock _l(_networks_m); + _networks[nwid].config = nlohmann::json::to_msgpack(j); + } + } else if ((id.length() == 10)&&(objtype == "member")) { + const uint64_t mid = Utils::strToU64(id.c_str()); + const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); + if ((mid)&&(nwid)) { + Mutex::Lock _l(_networks_m); + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); + } + } + } catch ( ... ) {} + } + } else { + this->_load((p + ZT_PATH_SEPARATOR_S + *di)); + } + } + return true; + + } +} + +void JSONDB::_recomputeSummaryInfo(const uint64_t networkId) +{ + Mutex::Lock _l(_summaryThread_m); + if (std::find(_summaryThreadToDo.begin(),_summaryThreadToDo.end(),networkId) == _summaryThreadToDo.end()) + _summaryThreadToDo.push_back(networkId); + if (!_summaryThread) + _summaryThread = Thread::start(this); +} + std::string JSONDB::_genPath(const std::string &n,bool create) { std::vector pt(OSUtils::split(n.c_str(),"/","","")); if (pt.size() == 0) return std::string(); + char sep; + if (_httpAddr) { + sep = '/'; + create = false; + } else { + sep = ZT_PATH_SEPARATOR; + } + std::string p(_basePath); if (create) OSUtils::mkdir(p.c_str()); for(unsigned long i=0,j=(unsigned long)(pt.size()-1);i #include #include +#include #include "../node/Constants.hpp" #include "../node/Utils.hpp" +#include "../node/InetAddress.hpp" +#include "../node/Mutex.hpp" #include "../ext/json/json.hpp" #include "../osdep/OSUtils.hpp" +#include "../osdep/Http.hpp" +#include "../osdep/Thread.hpp" namespace ZeroTier { /** - * Hierarchical JSON store that persists into the filesystem + * Hierarchical JSON store that persists into the filesystem or via HTTP */ class JSONDB { public: - JSONDB(const std::string &basePath) : - _basePath(basePath) + struct NetworkSummaryInfo { - _reload(_basePath,std::string()); - } + NetworkSummaryInfo() : authorizedMemberCount(0),activeMemberCount(0),totalMemberCount(0),mostRecentDeauthTime(0) {} + std::vector
activeBridges; + std::vector allocatedIps; + unsigned long authorizedMemberCount; + unsigned long activeMemberCount; + unsigned long totalMemberCount; + uint64_t mostRecentDeauthTime; + }; - inline void reload() - { - _db.clear(); - _reload(_basePath,std::string()); - } + JSONDB(const std::string &basePath); + ~JSONDB(); bool writeRaw(const std::string &n,const std::string &obj); - bool put(const std::string &n,const nlohmann::json &obj); + bool hasNetwork(const uint64_t networkId) const; - inline bool put(const std::string &n1,const std::string &n2,const nlohmann::json &obj) { return this->put((n1 + "/" + n2),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4),obj); } - inline bool put(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5,const nlohmann::json &obj) { return this->put((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5),obj); } + bool getNetwork(const uint64_t networkId,nlohmann::json &config) const; - const nlohmann::json &get(const std::string &n); + bool getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const; - inline const nlohmann::json &get(const std::string &n1,const std::string &n2) { return this->get((n1 + "/" + n2)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3) { return this->get((n1 + "/" + n2 + "/" + n3)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4)); } - inline const nlohmann::json &get(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { return this->get((n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5)); } + /** + * @return Bit mask: 0 == none, 1 == network only, 3 == network and member + */ + int getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const; - void erase(const std::string &n); + bool getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const; - inline void erase(const std::string &n1,const std::string &n2) { this->erase(n1 + "/" + n2); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3) { this->erase(n1 + "/" + n2 + "/" + n3); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4); } - inline void erase(const std::string &n1,const std::string &n2,const std::string &n3,const std::string &n4,const std::string &n5) { this->erase(n1 + "/" + n2 + "/" + n3 + "/" + n4 + "/" + n5); } + void saveNetwork(const uint64_t networkId,const nlohmann::json &networkConfig); + + void saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,const nlohmann::json &memberConfig); + + nlohmann::json eraseNetwork(const uint64_t networkId); + + nlohmann::json eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo = true); + + std::vector networkIds() const + { + std::vector r; + Mutex::Lock _l(_networks_m); + for(std::unordered_map::const_iterator n(_networks.begin());n!=_networks.end();++n) + r.push_back(n->first); + return r; + } template - inline void filter(const std::string &prefix,F func) + inline void eachMember(const uint64_t networkId,F func) { - for(std::map::iterator i(_db.lower_bound(prefix));i!=_db.end();) { - if ((i->first.length() >= prefix.length())&&(!memcmp(i->first.data(),prefix.data(),prefix.length()))) { - if (!func(i->first,get(i->first))) { - std::map::iterator i2(i); ++i2; - this->erase(i->first); - i = i2; - } else ++i; - } else break; + Mutex::Lock _l(_networks_m); + std::unordered_map::const_iterator i(_networks.find(networkId)); + if (i != _networks.end()) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + try { + func(networkId,m->first,nlohmann::json::from_msgpack(m->second)); + } catch ( ... ) {} + } } } - inline bool operator==(const JSONDB &db) const { return ((_basePath == db._basePath)&&(_db == db._db)); } - inline bool operator!=(const JSONDB &db) const { return (!(*this == db)); } + template + inline void eachId(F func) + { + Mutex::Lock _l(_networks_m); + for(std::unordered_map::const_iterator i(_networks.begin());i!=_networks.end();++i) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + try { + func(i->first,m->first); + } catch ( ... ) {} + } + } + } + + void threadMain() + throw(); private: - void _reload(const std::string &p,const std::string &b); - bool _isValidObjectName(const std::string &n); + bool _load(const std::string &p); + void _recomputeSummaryInfo(const uint64_t networkId); std::string _genPath(const std::string &n,bool create); - struct _E + std::string _basePath; + InetAddress _httpAddr; + + Thread _summaryThread; + std::vector _summaryThreadToDo; + volatile bool _summaryThreadRun; + Mutex _summaryThread_m; + + struct _NW { - nlohmann::json obj; - inline bool operator==(const _E &e) const { return (obj == e.obj); } - inline bool operator!=(const _E &e) const { return (obj != e.obj); } + _NW() : summaryInfoLastComputed(0) {} + std::vector config; + NetworkSummaryInfo summaryInfo; + uint64_t summaryInfoLastComputed; + std::unordered_map< uint64_t,std::vector > members; }; - std::string _basePath; - std::map _db; + std::unordered_map _networks; + Mutex _networks_m; }; } // namespace ZeroTier diff --git a/zto/controller/README.md b/zto/controller/README.md index db8d015..3519eb1 100644 --- a/zto/controller/README.md +++ b/zto/controller/README.md @@ -227,22 +227,12 @@ This returns an object containing all currently online members and the most rece | activeBridge | boolean | Member is able to bridge to other Ethernet nets | YES | | identity | string | Member's public ZeroTier identity (if known) | no | | ipAssignments | array[string] | Managed IP address assignments | YES | -| memberRevision | integer | Member revision counter | no | -| recentLog | array[object] | Recent member activity log; see below | no | +| revision | integer | Member revision counter | no | +| vMajor | integer | Most recently known major version | no | +| vMinor | integer | Most recently known minor version | no | +| vRev | integer | Most recently known revision | no | +| vProto | integer | Most recently known protocl version | no | +| physicalAddr | string | Last known physical IP/port or null if none | no | Note that managed IP assignments are only used if they fall within a managed route. Otherwise they are ignored. -**Recent log object format:** - -| Field | Type | Description | -| --------------------- | ------------- | ------------------------------------------------- | -| ts | integer | Time of request, ms since epoch | -| auth | boolean | Was member authorized? | -| authBy | string | How was member authorized? | -| vMajor | integer | Client major version or -1 if unknown | -| vMinor | integer | Client minor version or -1 if unknown | -| vRev | integer | Client revision or -1 if unknown | -| vProto | integer | ZeroTier protocol version reported by client | -| fromAddr | string | Physical address if known | - -The controller can only know a member's `fromAddr` if it's able to establish a direct path to it. Members behind very restrictive firewalls may not have this information since the controller will be receiving the member's requests by way of a relay. ZeroTier does not back-trace IP paths as packets are relayed since this would add a lot of protocol overhead. diff --git a/zto/controller/migrate-sqlite/migrate.js b/zto/controller/migrate-sqlite/migrate.js index ac9678a..2f2462c 100644 --- a/zto/controller/migrate-sqlite/migrate.js +++ b/zto/controller/migrate-sqlite/migrate.js @@ -95,7 +95,6 @@ async.series([function(nextStep) { creationTime: parseInt(row.creationTime)||0, enableBroadcast: !!row.enableBroadcast, ipAssignmentPools: [], - lastModified: Date.now(), multicastLimit: row.multicastLimit||32, name: row.name||'', private: !!row.private, @@ -177,7 +176,6 @@ async.series([function(nextStep) { ipAssignments: [], lastAuthorizedTime: (row.authorized) ? Date.now() : 0, lastDeauthorizedTime: (row.authorized) ? 0 : Date.now(), - lastModified: Date.now(), lastRequestMetaData: '', noAutoAssignIps: false, nwid: row.networkId, diff --git a/zto/ext/arm32-neon-salsa2012-asm/README.md b/zto/ext/arm32-neon-salsa2012-asm/README.md new file mode 100644 index 0000000..54fc6f5 --- /dev/null +++ b/zto/ext/arm32-neon-salsa2012-asm/README.md @@ -0,0 +1,6 @@ +ARM NEON (32-bit) ASM implementation of Salsa20/12 +====== + +This is from [supercop](http://bench.cr.yp.to/supercop.html) and was originally written by Daniel J. Bernstein. Code is in the public domain like the rest of Salsa20. It's much faster than the naive implementation. + +It's included automatically in 32-bit Linux ARM builds. It likely will not work on 64-bit ARM, so it'll need to be ported at least. That will unfortunately keep it out of mobile versions for now since those are all going 64-bit. diff --git a/zto/ext/arm32-neon-salsa2012-asm/salsa2012.h b/zto/ext/arm32-neon-salsa2012-asm/salsa2012.h new file mode 100644 index 0000000..95b247f --- /dev/null +++ b/zto/ext/arm32-neon-salsa2012-asm/salsa2012.h @@ -0,0 +1,23 @@ +#ifndef ZT_SALSA2012_ARM32NEON_ASM +#define ZT_SALSA2012_ARM32NEON_ASM + +#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux) +#include +#include +#define zt_arm_has_neon() ((getauxval(AT_HWCAP) & HWCAP_NEON) != 0) +#else +#define zt_arm_has_neon() (true) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// ciphertext buffer, message/NULL, length, nonce (8 bytes), key (32 bytes) +extern int zt_salsa2012_armneon3_xor(unsigned char *c,const unsigned char *m,unsigned long long len,const unsigned char *n,const unsigned char *k); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/zto/ext/arm32-neon-salsa2012-asm/salsa2012.s b/zto/ext/arm32-neon-salsa2012-asm/salsa2012.s new file mode 100644 index 0000000..9e5989c --- /dev/null +++ b/zto/ext/arm32-neon-salsa2012-asm/salsa2012.s @@ -0,0 +1,2231 @@ + +# qhasm: int32 input_0 + +# qhasm: int32 input_1 + +# qhasm: int32 input_2 + +# qhasm: int32 input_3 + +# qhasm: stack32 input_4 + +# qhasm: stack32 input_5 + +# qhasm: stack32 input_6 + +# qhasm: stack32 input_7 + +# qhasm: int32 caller_r4 + +# qhasm: int32 caller_r5 + +# qhasm: int32 caller_r6 + +# qhasm: int32 caller_r7 + +# qhasm: int32 caller_r8 + +# qhasm: int32 caller_r9 + +# qhasm: int32 caller_r10 + +# qhasm: int32 caller_r11 + +# qhasm: int32 caller_r14 + +# qhasm: reg128 caller_q4 + +# qhasm: reg128 caller_q5 + +# qhasm: reg128 caller_q6 + +# qhasm: reg128 caller_q7 + +# qhasm: startcode +.fpu neon +.text + +# qhasm: constant sigma: +.align 2 +sigma: + +# qhasm: const32 1634760805 +.word 1634760805 + +# qhasm: const32 857760878 +.word 857760878 + +# qhasm: const32 2036477234 +.word 2036477234 + +# qhasm: const32 1797285236 +.word 1797285236 + +# qhasm: int128 abab + +# qhasm: int128 diag0 + +# qhasm: int128 diag1 + +# qhasm: int128 diag2 + +# qhasm: int128 diag3 + +# qhasm: int128 a0 + +# qhasm: int128 a1 + +# qhasm: int128 a2 + +# qhasm: int128 a3 + +# qhasm: int128 b0 + +# qhasm: int128 b1 + +# qhasm: int128 b2 + +# qhasm: int128 b3 + +# qhasm: int128 next_diag0 + +# qhasm: int128 next_diag1 + +# qhasm: int128 next_diag2 + +# qhasm: int128 next_diag3 + +# qhasm: int128 next_a0 + +# qhasm: int128 next_a1 + +# qhasm: int128 next_a2 + +# qhasm: int128 next_a3 + +# qhasm: int128 next_b0 + +# qhasm: int128 next_b1 + +# qhasm: int128 next_b2 + +# qhasm: int128 next_b3 + +# qhasm: int128 x0x5x10x15 + +# qhasm: int128 x12x1x6x11 + +# qhasm: int128 x8x13x2x7 + +# qhasm: int128 x4x9x14x3 + +# qhasm: int128 x0x1x10x11 + +# qhasm: int128 x12x13x6x7 + +# qhasm: int128 x8x9x2x3 + +# qhasm: int128 x4x5x14x15 + +# qhasm: int128 x0x1x2x3 + +# qhasm: int128 x4x5x6x7 + +# qhasm: int128 x8x9x10x11 + +# qhasm: int128 x12x13x14x15 + +# qhasm: int128 m0m1m2m3 + +# qhasm: int128 m4m5m6m7 + +# qhasm: int128 m8m9m10m11 + +# qhasm: int128 m12m13m14m15 + +# qhasm: int128 start0 + +# qhasm: int128 start1 + +# qhasm: int128 start2 + +# qhasm: int128 start3 + +# qhasm: stack128 stack_start3 + +# qhasm: stack128 next_start2 + +# qhasm: stack128 next_start3 + +# qhasm: int128 k0k1k2k3 + +# qhasm: int128 k4k5k6k7 + +# qhasm: int128 k1n1k7k2 + +# qhasm: int128 n2n3n3n2 + +# qhasm: int128 k2k3k6k7 + +# qhasm: int128 nextblock + +# qhasm: stack128 stack_q4 + +# qhasm: stack128 stack_q5 + +# qhasm: stack128 stack_q6 + +# qhasm: stack128 stack_q7 + +# qhasm: stack32 stack_r4 + +# qhasm: stack128 k2k3k6k7_stack + +# qhasm: stack128 k1n1k7k2_stack + +# qhasm: stack512 tmp + +# qhasm: stack32 savec + +# qhasm: int32 i + +# qhasm: int32 ci + +# qhasm: int32 mi + +# qhasm: enter zt_salsa2012_armneon3_xor +.align 2 +.global _zt_salsa2012_armneon3_xor +.global zt_salsa2012_armneon3_xor +.type _zt_salsa2012_armneon3_xor STT_FUNC +.type zt_salsa2012_armneon3_xor STT_FUNC +_zt_salsa2012_armneon3_xor: +zt_salsa2012_armneon3_xor: +sub sp,sp,#256 + +# qhasm: new stack_q4 + +# qhasm: new stack_q5 + +# qhasm: new stack_q6 + +# qhasm: new stack_q7 + +# qhasm: stack_q4 bot = caller_q4 bot +# asm 1: vstr stack_r4=stack32#2 +# asm 2: str stack_r4=[sp,#68] +str r4,[sp,#68] + +# qhasm: int32 c + +# qhasm: c = input_0 +# asm 1: mov >c=int32#1,c=r0,m=int32#2,m=r1,mlenlow=int32#3,mlenlow=r2,mlenhigh=int32#4,mlenhigh=r3,n=int32#5,n=r4,k=int32#13,k=r12,k0k1k2k3=reg128#1%bot->k0k1k2k3=reg128#1%top},[k0k1k2k3=d0->k0k1k2k3=d1},[k4k5k6k7=reg128#2%bot->k4k5k6k7=reg128#2%top},[k4k5k6k7=d2->k4k5k6k7=d3},[i=int32#13,=sigma +# asm 2: ldr >i=r12,=sigma +ldr r12,=sigma + +# qhasm: start0 = mem128[i] +# asm 1: vld1.8 {>start0=reg128#3%bot->start0=reg128#3%top},[start0=d4->start0=d5},[start1=reg128#4,#0 +# asm 2: vmov.i64 >start1=q3,#0 +vmov.i64 q3,#0 + +# qhasm: start1 bot = mem64[n] +# asm 1: vld1.8 {k2k3k6k7=reg128#6,k2k3k6k7=q5,n2n3n3n2=reg128#1,#0 +# asm 2: vmov.i64 >n2n3n3n2=q0,#0 +vmov.i64 q0,#0 + +# qhasm: unsigneddiag0=reg128#8,diag0=q7,diag1=reg128#9,diag1=q8,start2=reg128#10,start2=q9,nextblock=reg128#11,#0xff +# asm 2: vmov.i64 >nextblock=q10,#0xff +vmov.i64 q10,#0xff + +# qhasm: 4x nextblock unsigned>>= 7 +# asm 1: vshr.u32 >nextblock=reg128#11,nextblock=q10,n2n3n3n2=reg128#1,n2n3n3n2=q0,n2n3n3n2=reg128#1,n2n3n3n2=q0,next_diag0=reg128#2,next_diag0=q1,next_diag1=reg128#5,next_diag1=q4,i=int32#5,=12 +# asm 2: ldr >i=r4,=12 +ldr r4,=12 + +# qhasm: mainloop2: +._mainloop2: + +# qhasm: 4x a0 = diag1 + diag0 +# asm 1: vadd.i32 >a0=reg128#11,a0=q10,next_a0=reg128#14,next_a0=q13,b0=reg128#15,b0=q14,next_b0=reg128#16,next_b0=q15,> 25 +# asm 1: vsri.i32 > 25 +# asm 1: vsri.i32 diag3=reg128#7,diag3=q6,next_diag3=reg128#11,next_diag3=q10,a1=reg128#13,a1=q12,next_a1=reg128#14,next_a1=q13,b1=reg128#15,b1=q14,next_b1=reg128#16,next_b1=q15,> 23 +# asm 1: vsri.i32 > 23 +# asm 1: vsri.i32 diag2=reg128#6,diag2=q5,next_diag2=reg128#12,next_diag2=q11,a2=reg128#13,a2=q12,diag3=reg128#7,diag3=q6,next_a2=reg128#14,next_a2=q13,b2=reg128#15,b2=q14,next_diag3=reg128#11,next_diag3=q10,next_b2=reg128#16,next_b2=q15,> 19 +# asm 1: vsri.i32 > 19 +# asm 1: vsri.i32 diag1=reg128#9,diag1=q8,next_diag1=reg128#5,next_diag1=q4,a3=reg128#13,a3=q12,next_a3=reg128#14,next_a3=q13,b3=reg128#15,b3=q14,next_b3=reg128#16,next_b3=q15,> 14 +# asm 1: vsri.i32 diag1=reg128#9,diag1=q8,> 14 +# asm 1: vsri.i32 diag0=reg128#8,diag0=q7,next_diag1=reg128#5,next_diag1=q4,next_diag0=reg128#2,next_diag0=q1,a0=reg128#13,a0=q12,next_a0=reg128#14,next_a0=q13,b0=reg128#15,b0=q14,next_b0=reg128#16,next_b0=q15,> 25 +# asm 1: vsri.i32 > 25 +# asm 1: vsri.i32 diag1=reg128#9,diag1=q8,next_diag1=reg128#5,next_diag1=q4,a1=reg128#13,a1=q12,next_a1=reg128#14,next_a1=q13,b1=reg128#15,b1=q14,next_b1=reg128#16,next_b1=q15,> 23 +# asm 1: vsri.i32 ? i -= 2 +# asm 1: subs > 23 +# asm 1: vsri.i32 diag2=reg128#6,diag2=q5,next_diag2=reg128#12,next_diag2=q11,a2=reg128#13,a2=q12,diag1=reg128#9,diag1=q8,next_a2=reg128#14,next_a2=q13,b2=reg128#15,b2=q14,next_diag1=reg128#5,next_diag1=q4,next_b2=reg128#16,next_b2=q15,> 19 +# asm 1: vsri.i32 > 19 +# asm 1: vsri.i32 diag3=reg128#7,diag3=q6,next_diag3=reg128#11,next_diag3=q10,a3=reg128#13,a3=q12,next_a3=reg128#14,next_a3=q13,b3=reg128#15,b3=q14,next_b3=reg128#16,next_b3=q15,> 14 +# asm 1: vsri.i32 diag3=reg128#7,diag3=q6,> 14 +# asm 1: vsri.i32 diag0=reg128#8,diag0=q7,next_diag3=reg128#13,next_diag3=q12,next_diag0=reg128#2,next_diag0=q1, +bhi ._mainloop2 + +# qhasm: 2x abab = 0xffffffff +# asm 1: vmov.i64 >abab=reg128#11,#0xffffffff +# asm 2: vmov.i64 >abab=q10,#0xffffffff +vmov.i64 q10,#0xffffffff + +# qhasm: new x4x9x14x3 + +# qhasm: x4x9x14x3 bot = stack_start3 bot +# asm 1: vldr x0x5x10x15=reg128#8,x0x5x10x15=q7,x12x1x6x11=reg128#9,x12x1x6x11=q8,x8x13x2x7=reg128#6,x8x13x2x7=q5,x4x9x14x3=reg128#7,x4x9x14x3=q6,x0x1x10x11=reg128#10,x0x1x10x11=q9,x12x13x6x7=reg128#14,x12x13x6x7=q13,x8x9x2x3=reg128#15,x8x9x2x3=q14,x4x5x14x15=reg128#16,x4x5x14x15=q15,x0x1x2x3=reg128#6,x0x1x2x3=q5,x4x5x6x7=reg128#7,x4x5x6x7=q6,x8x9x10x11=reg128#8,x8x9x10x11=q7,x12x13x14x15=reg128#9,x12x13x14x15=q8,m0m1m2m3=reg128#10%bot->m0m1m2m3=reg128#10%top},[m0m1m2m3=d18->m0m1m2m3=d19},[m4m5m6m7=reg128#14%bot->m4m5m6m7=reg128#14%top},[m4m5m6m7=d26->m4m5m6m7=d27},[m8m9m10m11=reg128#15%bot->m8m9m10m11=reg128#15%top},[m8m9m10m11=d28->m8m9m10m11=d29},[m12m13m14m15=reg128#16%bot->m12m13m14m15=reg128#16%top},[m12m13m14m15=d30->m12m13m14m15=d31},[x0x1x2x3=reg128#6,x0x1x2x3=q5,x4x5x6x7=reg128#7,x4x5x6x7=q6,x8x9x10x11=reg128#8,x8x9x10x11=q7,x12x13x14x15=reg128#9,x12x13x14x15=q8,x0x5x10x15=reg128#2,x0x5x10x15=q1,x12x1x6x11=reg128#5,x12x1x6x11=q4,x8x13x2x7=reg128#6,x8x13x2x7=q5,x4x9x14x3=reg128#7,x4x9x14x3=q6,x0x1x10x11=reg128#8,x0x1x10x11=q7,x12x13x6x7=reg128#9,x12x13x6x7=q8,x8x9x2x3=reg128#10,x8x9x2x3=q9,x4x5x14x15=reg128#12,x4x5x14x15=q11,x0x1x2x3=reg128#2,x0x1x2x3=q1,x4x5x6x7=reg128#5,x4x5x6x7=q4,x8x9x10x11=reg128#6,x8x9x10x11=q5,x12x13x14x15=reg128#7,x12x13x14x15=q6,m0m1m2m3=reg128#8%bot->m0m1m2m3=reg128#8%top},[m0m1m2m3=d14->m0m1m2m3=d15},[m4m5m6m7=reg128#9%bot->m4m5m6m7=reg128#9%top},[m4m5m6m7=d16->m4m5m6m7=d17},[m8m9m10m11=reg128#10%bot->m8m9m10m11=reg128#10%top},[m8m9m10m11=d18->m8m9m10m11=d19},[m12m13m14m15=reg128#11%bot->m12m13m14m15=reg128#11%top},[m12m13m14m15=d20->m12m13m14m15=d21},[x0x1x2x3=reg128#2,x0x1x2x3=q1,x4x5x6x7=reg128#5,x4x5x6x7=q4,x8x9x10x11=reg128#6,x8x9x10x11=q5,x12x13x14x15=reg128#7,x12x13x14x15=q6,? mlenhigh - 0 +# asm 1: cmp +bhi ._mlenatleast128 + +# qhasm: =? mlenlow - 0 +# asm 1: cmp savec=stack32#1 +# asm 2: str savec=[sp,#64] +str r0,[sp,#64] + +# qhasm: c = &tmp +# asm 1: lea >c=int32#1,c=r0,i=int32#4,=0 +# asm 2: ldr >i=r3,=0 +ldr r3,=0 + +# qhasm: mcopy: +._mcopy: + +# qhasm: mi = mem8[m + 0] +# asm 1: ldrb >mi=int32#5,[mi=r4,[mi=int32#2,=0 +# asm 2: ldr >mi=r1,=0 +ldr r1,=0 + +# qhasm: pad: +._pad: + +# qhasm: mem8[c + 0] = mi +# asm 1: strb m=int32#2,m=r1,diag0=reg128#2,diag0=q1,diag1=reg128#5,diag1=q4,diag2=reg128#8,diag2=q7,diag3=reg128#9,diag3=q8,nextblock=reg128#10,#0xff +# asm 2: vmov.i64 >nextblock=q9,#0xff +vmov.i64 q9,#0xff + +# qhasm: 4x nextblock unsigned>>= 7 +# asm 1: vshr.u32 >nextblock=reg128#10,nextblock=q9,n2n3n3n2=reg128#1,n2n3n3n2=q0,i=int32#4,=12 +# asm 2: ldr >i=r3,=12 +ldr r3,=12 + +# qhasm: mainloop1: +._mainloop1: + +# qhasm: 4x a0 = diag1 + diag0 +# asm 1: vadd.i32 >a0=reg128#10,a0=q9,b0=reg128#11,b0=q10,> 25 +# asm 1: vsri.i32 diag3=reg128#9,diag3=q8,a1=reg128#10,a1=q9,b1=reg128#11,b1=q10,> 23 +# asm 1: vsri.i32 diag2=reg128#8,diag2=q7,a2=reg128#10,a2=q9,diag3=reg128#9,diag3=q8,b2=reg128#11,b2=q10,> 19 +# asm 1: vsri.i32 diag1=reg128#5,diag1=q4,a3=reg128#10,a3=q9,b3=reg128#11,b3=q10,> 14 +# asm 1: vsri.i32 diag1=reg128#5,diag1=q4,diag0=reg128#2,diag0=q1,a0=reg128#10,a0=q9,b0=reg128#11,b0=q10,> 25 +# asm 1: vsri.i32 diag1=reg128#5,diag1=q4,a1=reg128#10,a1=q9,b1=reg128#11,b1=q10,> 23 +# asm 1: vsri.i32 ? i -= 2 +# asm 1: subs diag2=reg128#8,diag2=q7,a2=reg128#10,a2=q9,diag1=reg128#5,diag1=q4,b2=reg128#11,b2=q10,> 19 +# asm 1: vsri.i32 diag3=reg128#9,diag3=q8,a3=reg128#10,a3=q9,b3=reg128#11,b3=q10,> 14 +# asm 1: vsri.i32 diag3=reg128#9,diag3=q8,diag0=reg128#2,diag0=q1, +bhi ._mainloop1 + +# qhasm: 2x abab = 0xffffffff +# asm 1: vmov.i64 >abab=reg128#10,#0xffffffff +# asm 2: vmov.i64 >abab=q9,#0xffffffff +vmov.i64 q9,#0xffffffff + +# qhasm: 4x x0x5x10x15 = diag0 + start0 +# asm 1: vadd.i32 >x0x5x10x15=reg128#2,x0x5x10x15=q1,x12x1x6x11=reg128#5,x12x1x6x11=q4,x8x13x2x7=reg128#6,x8x13x2x7=q5,x4x9x14x3=reg128#7,x4x9x14x3=q6,x0x1x10x11=reg128#8,x0x1x10x11=q7,x12x13x6x7=reg128#9,x12x13x6x7=q8,x8x9x2x3=reg128#11,x8x9x2x3=q10,x4x5x14x15=reg128#12,x4x5x14x15=q11,x0x1x2x3=reg128#2,x0x1x2x3=q1,x4x5x6x7=reg128#5,x4x5x6x7=q4,x8x9x10x11=reg128#6,x8x9x10x11=q5,x12x13x14x15=reg128#7,x12x13x14x15=q6,m0m1m2m3=reg128#8%bot->m0m1m2m3=reg128#8%top},[m0m1m2m3=d14->m0m1m2m3=d15},[m4m5m6m7=reg128#9%bot->m4m5m6m7=reg128#9%top},[m4m5m6m7=d16->m4m5m6m7=d17},[m8m9m10m11=reg128#10%bot->m8m9m10m11=reg128#10%top},[m8m9m10m11=d18->m8m9m10m11=d19},[m12m13m14m15=reg128#11%bot->m12m13m14m15=reg128#11%top},[m12m13m14m15=d20->m12m13m14m15=d21},[x0x1x2x3=reg128#2,x0x1x2x3=q1,x4x5x6x7=reg128#5,x4x5x6x7=q4,x8x9x10x11=reg128#6,x8x9x10x11=q5,x12x13x14x15=reg128#7,x12x13x14x15=q6,i=int32#4,=0 +# asm 2: ldr >i=r3,=0 +ldr r3,=0 + +# qhasm: m = c - 64 +# asm 1: sub >m=int32#2,m=r1,c=int32#1,c=r0,ci=int32#5,[ci=r4,[? mlenlow -= 64 +# asm 1: subs +bhi ._mlenatleast1 + +# qhasm: done: +._done: + +# qhasm: new caller_r4 + +# qhasm: caller_r4 = stack_r4 +# asm 1: ldr >caller_r4=int32#5,caller_r4=r4,result=int32#1,=0 +# asm 2: ldr >result=r0,=0 +ldr r0,=0 + +# qhasm: return result +add sp,sp,#256 +bx lr diff --git a/zto/ext/installfiles/linux/zerotier-containerized/Dockerfile b/zto/ext/installfiles/linux/zerotier-containerized/Dockerfile new file mode 100644 index 0000000..678216d --- /dev/null +++ b/zto/ext/installfiles/linux/zerotier-containerized/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:latest +MAINTAINER Adam Ierymenko + +LABEL version="1.1.14" +LABEL description="Containerized ZeroTier One for use on CoreOS or other Docker-only Linux hosts." + +# Uncomment to build in container +#RUN apk add --update alpine-sdk linux-headers + +RUN apk add --update libgcc libstdc++ + +ADD zerotier-one / +RUN chmod 0755 /zerotier-one +RUN ln -sf /zerotier-one /zerotier-cli +RUN mkdir -p /var/lib/zerotier-one + +ADD main.sh / +RUN chmod 0755 /main.sh + +ENTRYPOINT /main.sh diff --git a/zto/ext/installfiles/linux/zerotier-containerized/main.sh b/zto/ext/installfiles/linux/zerotier-containerized/main.sh new file mode 100755 index 0000000..685a689 --- /dev/null +++ b/zto/ext/installfiles/linux/zerotier-containerized/main.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin + +if [ ! -e /dev/net/tun ]; then + echo 'FATAL: cannot start ZeroTier One in container: /dev/net/tun not present.' + exit 1 +fi + +exec /zerotier-one diff --git a/zto/ext/installfiles/mac/ZeroTier One.pkgproj b/zto/ext/installfiles/mac/ZeroTier One.pkgproj index c41b61c..96b1338 100755 --- a/zto/ext/installfiles/mac/ZeroTier One.pkgproj +++ b/zto/ext/installfiles/mac/ZeroTier One.pkgproj @@ -646,7 +646,7 @@ OVERWRITE_PERMISSIONS VERSION - 1.2.2 + 1.2.4 PROJECT_COMMENTS diff --git a/zto/ext/installfiles/mac/postinst.sh b/zto/ext/installfiles/mac/postinst.sh index 2e4f591..4f5a50c 100755 --- a/zto/ext/installfiles/mac/postinst.sh +++ b/zto/ext/installfiles/mac/postinst.sh @@ -43,4 +43,13 @@ ln -sf "/Library/Application Support/ZeroTier/One/zerotier-one" zerotier-idtool launchctl load /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1 +sleep 1 + +if [ -f /tmp/zt1-gui-restart.tmp ]; + for u in `cat /tmp/zt1-gui-restart.tmp`; do + su $u -c '/Applications/ZeroTier\ One.app/Contents/MacOS/ZeroTier\ One &' >>/dev/null 2>&1 & + done +fi +rm -f /tmp/zt1-gui-restart.tmp + exit 0 diff --git a/zto/ext/installfiles/mac/preinst.sh b/zto/ext/installfiles/mac/preinst.sh index c2cb494..af2a932 100755 --- a/zto/ext/installfiles/mac/preinst.sh +++ b/zto/ext/installfiles/mac/preinst.sh @@ -2,6 +2,19 @@ export PATH=/bin:/usr/bin:/sbin:/usr/sbin +rm -f /tmp/zt1-gui-restart.tmp +for i in `ps axuwww | tr -s ' ' ',' | grep -F '/Applications/ZeroTier,One.app' | grep -F -v grep | cut -d , -f 1,2 | xargs`; do + u=`echo $i | cut -d , -f 1` + p=`echo $i | cut -d , -f 2` + if [ ! -z "$u" -a "0$p" -gt 0 ]; then + kill $p >>/dev/null 2>&1 + sleep 0.2 + kill -9 $p >>/dev/null 2>&1 + echo "$u" >>/tmp/zt1-gui-restart.tmp + fi +done +chmod 0600 /tmp/zt1-gui-restart.tmp + if [ -f /Library/LaunchDaemons/com.zerotier.one.plist ]; then launchctl unload /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1 fi diff --git a/zto/ext/installfiles/mac/uninstall.sh b/zto/ext/installfiles/mac/uninstall.sh index d1effb9..9bf5d6f 100755 --- a/zto/ext/installfiles/mac/uninstall.sh +++ b/zto/ext/installfiles/mac/uninstall.sh @@ -28,13 +28,11 @@ echo "Removing ZeroTier One files..." rm -rf '/Applications/ZeroTier One.app' rm -f '/usr/bin/zerotier-one' '/usr/bin/zerotier-idtool' '/usr/bin/zerotier-cli' '/Library/LaunchDaemons/com.zerotier.one.plist' -mkdir -p /tmp/ZeroTierOne_uninstall_tmp -cp "/Library/Application Support/ZeroTier/One/*.secret" /tmp/ZeroTierOne_uninstall_tmp -rm -rf '/Library/Application Support/ZeroTier/One' -mkdir -p '/Library/Application Support/ZeroTier/One' -cp "/tmp/ZeroTierOne_uninstall_tmp/*.secret" '/Library/Application Support/ZeroTier/One' -chmod 0600 "/Library/Application Support/ZeroTier/One/*.secret" -rm -rf /tmp/ZeroTierOne_uninstall_tmp + +cd '/Library/Application Support/ZeroTier/One' +if [ "`pwd`" = '/Library/Application Support/ZeroTier/One' ]; then + rm -rf *.d *.sh *.log *.old *.kext *.conf *.pkg *.dmg *.pid *.port *.save *.bin planet zerotier-* devicemap +fi echo 'Uninstall complete.' echo diff --git a/zto/ext/installfiles/windows/ZeroTier One.aip b/zto/ext/installfiles/windows/ZeroTier One.aip index 29a02de..a63fa2b 100644 --- a/zto/ext/installfiles/windows/ZeroTier One.aip +++ b/zto/ext/installfiles/windows/ZeroTier One.aip @@ -27,10 +27,10 @@ - + - + @@ -60,7 +60,7 @@ - + @@ -432,7 +432,7 @@ - + diff --git a/zto/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec b/zto/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec index 03987e3..32fa5a9 100644 --- a/zto/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec +++ b/zto/ext/installfiles/windows/chocolatey/zerotier-one/zerotier-one.nuspec @@ -26,7 +26,7 @@ This is a nuspec. It mostly adheres to https://docs.nuget.org/create/Nuspec-Refe - 1.2.2 + 1.2.4 diff --git a/zto/ext/json/LICENSE.MIT b/zto/ext/json/LICENSE.MIT index e2ac489..00599af 100644 --- a/zto/ext/json/LICENSE.MIT +++ b/zto/ext/json/LICENSE.MIT @@ -1,14 +1,13 @@ -The library is licensed under the MIT License -: +MIT License -Copyright (c) 2013-2016 Niels Lohmann +Copyright (c) 2013-2017 Niels Lohmann -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. diff --git a/zto/ext/json/README.md b/zto/ext/json/README.md index 4bcbe97..fc6dde9 100644 --- a/zto/ext/json/README.md +++ b/zto/ext/json/README.md @@ -3,13 +3,34 @@ [![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json) [![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk/branch/develop?svg=true)](https://ci.appveyor.com/project/nlohmann/json) [![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json) -[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/fsf5FqYe6GoX68W6) +[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/f3732b3327e34358a0e9d1fe9f661f08)](https://www.codacy.com/app/nlohmann/json?utm_source=github.com&utm_medium=referral&utm_content=nlohmann/json&utm_campaign=Badge_Grade) +[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/nv9fOg0XVVhWmFFy) [![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT) [![Github Releases](https://img.shields.io/github/release/nlohmann/json.svg)](https://github.com/nlohmann/json/releases) [![Github Issues](https://img.shields.io/github/issues/nlohmann/json.svg)](http://github.com/nlohmann/json/issues) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/289/badge)](https://bestpractices.coreinfrastructure.org/projects/289) +- [Design goals](#design-goals) +- [Integration](#integration) +- [Examples](#examples) + - [JSON as first-class data type](#json-as-first-class-data-type) + - [Serialization / Deserialization](#serialization--deserialization) + - [STL-like access](#stl-like-access) + - [Conversion from STL containers](#conversion-from-stl-containers) + - [JSON Pointer and JSON Patch](#json-pointer-and-json-patch) + - [Implicit conversions](#implicit-conversions) + - [Conversions to/from arbitrary types](#arbitrary-types-conversions) + - [Binary formats (CBOR and MessagePack)](#binary-formats-cbor-and-messagepack) +- [Supported compilers](#supported-compilers) +- [License](#license) +- [Thanks](#thanks) +- [Used third-party tools](#used-third-party-tools) +- [Projects using JSON for Modern C++](#projects-using-json-for-modern-c) +- [Notes](#notes) +- [Execute unit tests](#execute-unit-tests) + ## Design goals There are myriads of [JSON](http://json.org) libraries out there, and each may even have its reason to exist. Our class had these design goals: @@ -24,7 +45,7 @@ Other aspects were not so important to us: - **Memory efficiency**. Each JSON object has an overhead of one pointer (the maximal size of a union) and one enumeration element (1 byte). The default generalization uses the following C++ data types: `std::string` for strings, `int64_t`, `uint64_t` or `double` for numbers, `std::map` for objects, `std::vector` for arrays, and `bool` for Booleans. However, you can template the generalized class `basic_json` to your needs. -- **Speed**. We currently implement the parser as naive [recursive descent parser](http://en.wikipedia.org/wiki/Recursive_descent_parser) with hand coded string handling. It is fast enough, but a [LALR-parser](http://en.wikipedia.org/wiki/LALR_parser) may be even faster (but would consist of more files which makes the integration harder). +- **Speed**. There are certainly [faster JSON libraries](https://github.com/miloyip/nativejson-benchmark#parsing-time) out there. However, if your goal is to speed up your development by adding JSON support with a single header, then this library is the way to go. If you know how to use a `std::vector` or `std::map`, you are already set. See the [contribution guidelines](https://github.com/nlohmann/json/blob/master/.github/CONTRIBUTING.md#please-dont) for more information. @@ -44,9 +65,15 @@ to the files you want to use JSON objects. That's it. Do not forget to set the n :beer: If you are using OS X and [Homebrew](http://brew.sh), just type `brew tap nlohmann/json` and `brew install nlohmann_json` and you're set. If you want the bleeding edge rather than the latest release, use `brew install nlohmann_json --HEAD`. +:warning: [Version 3.0.0](https://github.com/nlohmann/json/wiki/Road-toward-3.0.0) is currently under development. Branch `develop` is used for the ongoing work and is probably **unstable**. Please use the `master` branch for the last stable version 2.1.1. + ## Examples +Beside the examples below, you may want to check the [documentation](https://nlohmann.github.io/json/) where each function contains a separate code example (e.g., check out [`emplace()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a602f275f0359ab181221384989810604.html#a602f275f0359ab181221384989810604)). All [example files](https://github.com/nlohmann/json/tree/develop/doc/examples) can be compiled and executed on their own (e.g., file [emplace.cpp](https://github.com/nlohmann/json/blob/develop/doc/examples/emplace.cpp)). + +### JSON as first-class data type + Here are some examples to give you an idea how to use the class. Assume you want to create the JSON object @@ -129,6 +156,8 @@ json array_not_object = { json::array({"currency", "USD"}), json::array({"value" ### Serialization / Deserialization +#### To/from strings + You can create an object (deserialization) by appending `_json` to a string literal: ```cpp @@ -142,8 +171,14 @@ auto j2 = R"( "pi": 3.141 } )"_json; +``` -// or explicitly +Note that without appending the `_json` suffix, the passed string literal is not parsed, but just used as JSON string value. That is, `json j = "{ \"happy\": true, \"pi\": 3.141 }"` would just store the string `"{ "happy": true, "pi": 3.141 }"` rather than parsing the actual object. + +The above example can also be expressed explicitly using `json::parse()`: + +```cpp +// parse explicitly auto j3 = json::parse("{ \"happy\": true, \"pi\": 3.141 }"); ``` @@ -162,6 +197,8 @@ std::cout << j.dump(4) << std::endl; // } ``` +#### To/from streams (e.g. files, string streams) + You can also use streams to serialize and deserialize: ```cpp @@ -176,10 +213,37 @@ std::cout << j; std::cout << std::setw(4) << j << std::endl; ``` -These operators work for any subclasses of `std::istream` or `std::ostream`. +These operators work for any subclasses of `std::istream` or `std::ostream`. Here is the same example with files: + +```cpp +// read a JSON file +std::ifstream i("file.json"); +json j; +i >> j; + +// write prettified JSON to another file +std::ofstream o("pretty.json"); +o << std::setw(4) << j << std::endl; +``` Please note that setting the exception bit for `failbit` is inappropriate for this use case. It will result in program termination due to the `noexcept` specifier in use. +#### Read from iterator range + +You can also read JSON from an iterator range; that is, from any container accessible by iterators whose content is stored as contiguous byte sequence, for instance a `std::vector`: + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v.begin(), v.end()); +``` + +You may leave the iterators for the range [begin, end): + +```cpp +std::vector v = {'t', 'r', 'u', 'e'}; +json j = json::parse(v); +``` + ### STL-like access @@ -192,6 +256,9 @@ j.push_back("foo"); j.push_back(1); j.push_back(true); +// also use emplace_back +j.emplace_back(1.78); + // iterate the array for (json::iterator it = j.begin(); it != j.end(); ++it) { std::cout << *it << '\n'; @@ -207,6 +274,9 @@ const std::string tmp = j[0]; j[1] = 42; bool foo = j.at(2); +// comparison +j == "[\"foo\", 1, true]"_json; // true + // other stuff j.size(); // 3 entries j.empty(); // false @@ -221,15 +291,15 @@ j.is_object(); j.is_array(); j.is_string(); -// comparison -j == "[\"foo\", 1, true]"_json; // true - // create an object json o; o["foo"] = 23; o["bar"] = false; o["baz"] = 3.141; +// also use emplace +o.emplace("weather", "sunny"); + // special iterator member functions for objects for (json::iterator it = o.begin(); it != o.end(); ++it) { std::cout << it.key() << " : " << it.value() << "\n"; @@ -383,6 +453,252 @@ int vi = jn.get(); // etc. ``` +### Arbitrary types conversions + +Every type can be serialized in JSON, not just STL-containers and scalar types. Usually, you would do something along those lines: + +```cpp +namespace ns { + // a simple struct to model a person + struct person { + std::string name; + std::string address; + int age; + }; +} + +ns::person p = {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// convert to JSON: copy each value into the JSON object +json j; +j["name"] = p.name; +j["address"] = p.address; +j["age"] = p.age; + +// ... + +// convert from JSON: copy each value from the JSON object +ns::person p { + j["name"].get(), + j["address"].get(), + j["age"].get() +}; +``` + +It works, but that's quite a lot of boilerplate... Fortunately, there's a better way: + +```cpp +// create a person +ns::person p {"Ned Flanders", "744 Evergreen Terrace", 60}; + +// conversion: person -> json +json j = p; + +std::cout << j << std::endl; +// {"address":"744 Evergreen Terrace","age":60,"name":"Ned Flanders"} + +// conversion: json -> person +ns::person p2 = j; + +// that's it +assert(p == p2); +``` + +#### Basic usage + +To make this work with one of your types, you only need to provide two functions: + +```cpp +using nlohmann::json; + +namespace ns { + void to_json(json& j, const person& p) { + j = json{{"name", p.name}, {"address", p.address}, {"age", p.age}}; + } + + void from_json(const json& j, person& p) { + p.name = j.at("name").get(); + p.address = j.at("address").get(); + p.age = j.at("age").get(); + } +} // namespace ns +``` + +That's all! When calling the `json` constructor with your type, your custom `to_json` method will be automatically called. +Likewise, when calling `get()`, the `from_json` method will be called. + +Some important things: + +* Those methods **MUST** be in your type's namespace (which can be the global namespace), or the library will not be able to locate them (in this example, they are in namespace `ns`, where `person` is defined). +* When using `get()`, `your_type` **MUST** be [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). (There is a way to bypass this requirement described later.) +* In function `from_json`, use function [`at()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a93403e803947b86f4da2d1fb3345cf2c.html#a93403e803947b86f4da2d1fb3345cf2c) to access the object values rather than `operator[]`. In case a key does not exists, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior. + +#### How do I convert third-party types? + +This requires a bit more advanced technique. But first, let's see how this conversion mechanism works: + +The library uses **JSON Serializers** to convert types to json. +The default serializer for `nlohmann::json` is `nlohmann::adl_serializer` (ADL means [Argument-Dependent Lookup](http://en.cppreference.com/w/cpp/language/adl)). + +It is implemented like this (simplified): + +```cpp +template +struct adl_serializer { + static void to_json(json& j, const T& value) { + // calls the "to_json" method in T's namespace + } + + static void from_json(const json& j, T& value) { + // same thing, but with the "from_json" method + } +}; +``` + +This serializer works fine when you have control over the type's namespace. However, what about `boost::optional`, or `std::filesystem::path` (C++17)? Hijacking the `boost` namespace is pretty bad, and it's illegal to add something other than template specializations to `std`... + +To solve this, you need to add a specialization of `adl_serializer` to the `nlohmann` namespace, here's an example: + +```cpp +// partial specialization (full specialization works too) +namespace nlohmann { + template + struct adl_serializer> { + static void to_json(json& j, const boost::optional& opt) { + if (opt == boost::none) { + j = nullptr; + } else { + j = *opt; // this will call adl_serializer::to_json which will + // find the free function to_json in T's namespace! + } + } + + static void from_json(const json& j, boost::optional& opt) { + if (j.is_null()) { + opt = boost::none; + } else { + opt = j.get(); // same as above, but with + // adl_serializer::from_json + } + } + }; +} +``` + +#### How can I use `get()` for non-default constructible/non-copyable types? + +There is a way, if your type is [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible). You will need to specialize the `adl_serializer` as well, but with a special `from_json` overload: + +```cpp +struct move_only_type { + move_only_type() = delete; + move_only_type(int ii): i(ii) {} + move_only_type(const move_only_type&) = delete; + move_only_type(move_only_type&&) = default; + + int i; +}; + +namespace nlohmann { + template <> + struct adl_serializer { + // note: the return type is no longer 'void', and the method only takes + // one argument + static move_only_type from_json(const json& j) { + return {j.get()}; + } + + // Here's the catch! You must provide a to_json method! Otherwise you + // will not be able to convert move_only_type to json, since you fully + // specialized adl_serializer on that type + static void to_json(json& j, move_only_type t) { + j = t.i; + } + }; +} +``` + +#### Can I write my own serializer? (Advanced use) + +Yes. You might want to take a look at [`unit-udt.cpp`](https://github.com/nlohmann/json/blob/develop/test/src/unit-udt.cpp) in the test suite, to see a few examples. + +If you write your own serializer, you'll need to do a few things: + +* use a different `basic_json` alias than `nlohmann::json` (the last template parameter of `basic_json` is the `JSONSerializer`) +* use your `basic_json` alias (or a template parameter) in all your `to_json`/`from_json` methods +* use `nlohmann::to_json` and `nlohmann::from_json` when you need ADL + +Here is an example, without simplifications, that only accepts types with a size <= 32, and uses ADL. + +```cpp +// You should use void as a second template argument +// if you don't need compile-time checks on T +template::type> +struct less_than_32_serializer { + template + static void to_json(BasicJsonType& j, T value) { + // we want to use ADL, and call the correct to_json overload + using nlohmann::to_json; // this method is called by adl_serializer, + // this is where the magic happens + to_json(j, value); + } + + template + static void from_json(const BasicJsonType& j, T& value) { + // same thing here + using nlohmann::from_json; + from_json(j, value); + } +}; +``` + +Be **very** careful when reimplementing your serializer, you can stack overflow if you don't pay attention: + +```cpp +template +struct bad_serializer +{ + template + static void to_json(BasicJsonType& j, const T& value) { + // this calls BasicJsonType::json_serializer::to_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + j = value; + } + + template + static void to_json(const BasicJsonType& j, T& value) { + // this calls BasicJsonType::json_serializer::from_json(j, value); + // if BasicJsonType::json_serializer == bad_serializer ... oops! + value = j.template get(); // oops! + } +}; +``` + +### Binary formats (CBOR and MessagePack) + +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation) and [MessagePack](http://msgpack.org) to efficiently encode JSON values to byte vectors and to decode such vectors. + +```cpp +// create a JSON value +json j = R"({"compact": true, "schema": 0})"_json; + +// serialize to CBOR +std::vector v_cbor = json::to_cbor(j); + +// 0xa2, 0x67, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xf5, 0x66, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_cbor = json::from_cbor(v_cbor); + +// serialize to MessagePack +std::vector v_msgpack = json::to_msgpack(j); + +// 0x82, 0xa7, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xc3, 0xa6, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x00 + +// roundtrip +json j_from_msgpack = json::from_msgpack(v_msgpack); +``` + ## Supported compilers @@ -391,6 +707,7 @@ Though it's 2016 already, the support for C++11 is still a bit sparse. Currently - GCC 4.9 - 6.0 (and possibly later) - Clang 3.4 - 3.9 (and possibly later) - Microsoft Visual C++ 2015 / Build Tools 14.0.25123.0 (and possibly later) +- Microsoft Visual C++ 2017 / Build Tools 15.1.548.43366 (and possibly later) I would be happy to learn about other compilers/versions. @@ -423,16 +740,14 @@ The following compilers are currently used in continuous integration at [Travis] | Clang 3.7.1 | Ubuntu 14.04.4 LTS | clang version 3.7.1 (tags/RELEASE_371/final) | | Clang 3.8.0 | Ubuntu 14.04.4 LTS | clang version 3.8.0 (tags/RELEASE_380/final) | | Clang 3.8.1 | Ubuntu 14.04.4 LTS | clang version 3.8.1 (tags/RELEASE_381/final) | -| Clang Xcode 6.1 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.54) (based on LLVM 3.5svn) | -| Clang Xcode 6.2 | Darwin Kernel Version 13.4.0 (OSX 10.9.5) | Apple LLVM version 6.0 (clang-600.0.57) (based on LLVM 3.5svn) | -| Clang Xcode 6.3 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.49) (based on LLVM 3.6.0svn) | | Clang Xcode 6.4 | Darwin Kernel Version 14.3.0 (OSX 10.10.3) | Apple LLVM version 6.1.0 (clang-602.0.53) (based on LLVM 3.6.0svn) | -| Clang Xcode 7.1 | Darwin Kernel Version 14.5.0 (OSX 10.10.5) | Apple LLVM version 7.0.0 (clang-700.1.76) | -| Clang Xcode 7.2 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.0.2 (clang-700.1.81) | | Clang Xcode 7.3 | Darwin Kernel Version 15.0.0 (OSX 10.10.5) | Apple LLVM version 7.3.0 (clang-703.0.29) | -| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 (OSX 10.11.6) | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.0 | Darwin Kernel Version 15.6.0 | Apple LLVM version 8.0.0 (clang-800.0.38) | +| Clang Xcode 8.1 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.2 | Darwin Kernel Version 16.1.0 (macOS 10.12.1) | Apple LLVM version 8.0.0 (clang-800.0.42.1) | +| Clang Xcode 8.3 | Darwin Kernel Version 16.5.0 (macOS 10.12.4) | Apple LLVM version 8.1.0 (clang-802.0.38) | | Visual Studio 14 2015 | Windows Server 2012 R2 (x64) | Microsoft (R) Build Engine version 14.0.25123.0 | - +| Visual Studio 2017 | Windows Server 2016 | Microsoft (R) Build Engine version 15.1.548.43366 | ## License @@ -440,7 +755,7 @@ The following compilers are currently used in continuous integration at [Travis] The class is licensed under the [MIT License](http://opensource.org/licenses/MIT): -Copyright © 2013-2016 [Niels Lohmann](http://nlohmann.me) +Copyright © 2013-2017 [Niels Lohmann](http://nlohmann.me) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: @@ -465,7 +780,7 @@ I deeply appreciate the help of the following people. - [Eric Cornelius](https://github.com/EricMCornelius) pointed out a bug in the handling with NaN and infinity values. He also improved the performance of the string escaping. - [易思龙](https://github.com/likebeta) implemented a conversion from anonymous enums. - [kepkin](https://github.com/kepkin) patiently pushed forward the support for Microsoft Visual studio. -- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. +- [gregmarr](https://github.com/gregmarr) simplified the implementation of reverse iterators and helped with numerous hints and improvements. In particular, he pushed forward the implementation of user-defined types. - [Caio Luppi](https://github.com/caiovlp) fixed a bug in the Unicode handling. - [dariomt](https://github.com/dariomt) fixed some typos in the examples. - [Daniel Frey](https://github.com/d-frey) cleaned up some pointers and implemented exception-safe memory allocation. @@ -493,14 +808,70 @@ I deeply appreciate the help of the following people. - [duncanwerner](https://github.com/duncanwerner) found a really embarrassing performance regression in the 2.0.0 release. - [Damien](https://github.com/dtoma) fixed one of the last conversion warnings. - [Thomas Braun](https://github.com/t-b) fixed a warning in a test case. -- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). +- [Théo DELRIEU](https://github.com/theodelrieu) patiently and constructively oversaw the long way toward [iterator-range parsing](https://github.com/nlohmann/json/issues/290). He also implemented the magic behind the serialization/deserialization of user-defined types. - [Stefan](https://github.com/5tefan) fixed a minor issue in the documentation. - [Vasil Dimov](https://github.com/vasild) fixed the documentation regarding conversions from `std::multiset`. - [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion. -- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable. +- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable and added Visual Studio 17 to the build matrix. - [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file. +- [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function. +- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling, found some nice performance improvements in the parser, improved the benchmarking code, and realized locale-independent number parsing and printing. +- [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan. +- [Jared Grubb](https://github.com/jaredgrubb) silenced a nasty documentation warning. +- [Yixin Zhang](https://github.com/qwename) fixed an integer overflow check. +- [Bosswestfalen](https://github.com/Bosswestfalen) merged two iterator classes into a smaller one. +- [Daniel599](https://github.com/Daniel599) helped to get Travis execute the tests with Clang's sanitizers. +- [Jonathan Lee](https://github.com/vjon) fixed an example in the README file. +- [gnzlbg](https://github.com/gnzlbg) supported the implementation of user-defined types. +- [Alexej Harm](https://github.com/qis) helped to get the user-defined types working with Visual Studio. +- [Jared Grubb](https://github.com/jaredgrubb) supported the implementation of user-defined types. +- [EnricoBilla](https://github.com/EnricoBilla) noted a typo in an example. +- [Martin Hořeňovský](https://github.com/horenmar) found a way for a 2x speedup for the compilation time of the test suite. +- [ukhegg](https://github.com/ukhegg) found proposed an improvement for the examples section. +- [rswanson-ihi](https://github.com/rswanson-ihi) noted a typo in the README. +- [Mihai Stan](https://github.com/stanmihai4) fixed a bug in the comparison with `nullptr`s. +- [Tushar Maheshwari](https://github.com/tusharpm) added [cotire](https://github.com/sakra/cotire) support to speed up the compilation. +- [TedLyngmo](https://github.com/TedLyngmo) noted a typo in the README, removed unnecessary bit arithmetic, and fixed some `-Weffc++` warnings. +- [Krzysztof Woś](https://github.com/krzysztofwos) made exceptions more visible. +- [ftillier](https://github.com/ftillier) fixed a compiler warning. +- [tinloaf](https://github.com/tinloaf) made sure all pushed warnings are properly popped. +- [Fytch](https://github.com/Fytch) found a bug in the documentation. -Thanks a lot for helping out! +Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone. + + +## Used third-party tools + +The library itself contains of a single header file licensed under the MIT license. However, it is built, tested, documented, and whatnot using a lot of third-party tools and services. Thanks a lot! + +- [**American fuzzy lop**](http://lcamtuf.coredump.cx/afl/) for fuzz testing +- [**AppVeyor**](https://www.appveyor.com) for [continuous integration](https://ci.appveyor.com/project/nlohmann/json) on Windows +- [**Artistic Style**](http://astyle.sourceforge.net) for automatic source code identation +- [**benchpress**](https://github.com/sbs-ableton/benchpress) to benchmark the code +- [**Catch**](https://github.com/philsquared/Catch) for the unit tests +- [**Clang**](http://clang.llvm.org) for compilation with code sanitizers +- [**Cmake**](https://cmake.org) for build automation +- [**Codacity**](https://www.codacy.com) for further [code analysis](https://www.codacy.com/app/nlohmann/json) +- [**cotire**](https://github.com/sakra/cotire) to speed of compilation +- [**Coveralls**](https://coveralls.io) to measure [code coverage](https://coveralls.io/github/nlohmann/json) +- [**Coverity Scan**](https://scan.coverity.com) for [static analysis](https://scan.coverity.com/projects/nlohmann-json) +- [**cppcheck**](http://cppcheck.sourceforge.net) for static analysis +- [**cxxopts**](https://github.com/jarro2783/cxxopts) to let benchpress parse command-line parameters +- [**Doxygen**](http://www.stack.nl/~dimitri/doxygen/) to generate [documentation](https://nlohmann.github.io/json/) +- [**git-update-ghpages**](https://github.com/rstacruz/git-update-ghpages) to upload the documentation to gh-pages +- [**Github Changelog Generator**](https://github.com/skywinder/github-changelog-generator) to generate the [ChangeLog](https://github.com/nlohmann/json/blob/develop/ChangeLog.md) +- [**libFuzzer**](http://llvm.org/docs/LibFuzzer.html) to implement fuzz testing for OSS-Fuzz +- [**OSS-Fuzz**](https://github.com/google/oss-fuzz) for continuous fuzz testing of the library +- [**re2c**](http://re2c.org) to generate an automaton for the lexical analysis +- [**send_to_wandbox**](https://github.com/nlohmann/json/blob/develop/doc/scripts/send_to_wandbox.py) to send code examples to [Wandbox](http://melpon.org/wandbox) +- [**Travis**](https://travis-ci.org) for [continuous integration](https://travis-ci.org/nlohmann/json) on Linux and macOS +- [**Valgrind**](http://valgrind.org) to check for correct memory management +- [**Wandbox**](http://melpon.org/wandbox) for [online examples](http://melpon.org/wandbox/permlink/4NEU6ZZMoM9lpIex) + + +## Projects using JSON for Modern C++ + +The library is currently used in Apple macOS Sierra and iOS 10. I am not sure what they are using the library for, but I am happy that it runs on so many devices. ## Notes @@ -512,6 +883,10 @@ Thanks a lot for helping out! - Other encodings such as Latin-1, UTF-16, or UTF-32 are not supported and will yield parse errors. - [Unicode noncharacters](http://www.unicode.org/faq/private_use.html#nonchar1) will not be replaced by the library. - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors. + - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs. +- The code can be compiled without C++ **runtime type identification** features; that is, you can use the `-fno-rtti` compiler flag. +- **Exceptions** are used widely within the library. They can, however, be switched off with either using the compiler flag `-fno-exceptions` or by defining the symbol `JSON_NOEXCEPTION`. In this case, exceptions are replaced by an `abort()` call. +- By default, the library does not preserve the **insertion order of object elements**. This is standards-compliant, as the [JSON standard](https://tools.ietf.org/html/rfc7159.html) defines objects as "an unordered collection of zero or more name/value pairs". If you do want to preserve the insertion order, you can specialize the object type with containers like [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) or [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map). ## Execute unit tests @@ -519,10 +894,11 @@ Thanks a lot for helping out! To compile and run the tests, you need to execute ```sh -$ make check +$ make json_unit -Ctest +$ ./test/json_unit "*" =============================================================================== -All tests passed (8905491 assertions in 36 test cases) +All tests passed (11203022 assertions in 48 test cases) ``` Alternatively, you can use [CMake](https://cmake.org) and run diff --git a/zto/ext/json/json.hpp b/zto/ext/json/json.hpp index 9d48e7a..8a8b876 100644 --- a/zto/ext/json/json.hpp +++ b/zto/ext/json/json.hpp @@ -1,7 +1,7 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 2.0.10 +| | |__ | | | | | | version 2.1.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . @@ -29,42 +29,39 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include // all_of, for_each, transform +#include // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform #include // array #include // assert -#include // isdigit #include // and, not, or -#include // isfinite, ldexp, signbit +#include // lconv, localeconv +#include // isfinite, labs, ldexp, signbit #include // nullptr_t, ptrdiff_t, size_t #include // int64_t, uint64_t -#include // strtod, strtof, strtold, strtoul +#include // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull #include // strlen +#include // forward_list #include // function, hash, less #include // initializer_list -#include // setw #include // istream, ostream -#include // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator +#include // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator #include // numeric_limits #include // locale #include // map #include // addressof, allocator, allocator_traits, unique_ptr #include // accumulate #include // stringstream -#include // domain_error, invalid_argument, out_of_range #include // getline, stoi, string, to_string -#include // add_pointer, enable_if, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_floating_point, is_integral, is_nothrow_move_assignable, std::is_nothrow_move_constructible, std::is_pointer, std::is_reference, std::is_same, remove_const, remove_pointer, remove_reference +#include // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type #include // declval, forward, make_pair, move, pair, swap #include // vector // exclude unsupported compilers #if defined(__clang__) - #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) - #if CLANG_VERSION < 30400 + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) - #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - #if GCC_VERSION < 40900 + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif @@ -90,6 +87,17 @@ SOFTWARE. #define JSON_DEPRECATED #endif +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) +#endif + /*! @brief namespace for Niels Lohmann @see https://github.com/nlohmann @@ -98,38 +106,1093 @@ SOFTWARE. namespace nlohmann { - /*! @brief unnamed namespace with internal helper functions + +This namespace collects some functions that could not be defined inside the +@ref basic_json class. + +@since version 2.1.0 +*/ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/*! +@brief general exception of the @ref basic_json class + +Extension of std::exception objects with a member @a id for exception ids. + +@note To have nothrow-copy-constructible exceptions, we internally use + std::runtime_error which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. + +@since version 3.0.0 +*/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + virtual const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) + : id(id_), m(what_arg) + {} + + static std::string name(const std::string& ename, int id) + { + return "[json.exception." + ename + "." + std::to_string(id) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/*! +@brief exception indicating a parse error + +This excpetion is thrown by the library when a parse error occurs. Parse +errors can occur during the deserialization of JSON text as well as when +using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +@note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + +Exceptions have ids 1xx. + +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.111 | parse error: bad input stream | Parsing CBOR or MessagePack from an input stream where the [`badbit` or `failbit`](http://en.cppreference.com/w/cpp/io/ios_base/iostate) is set. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. + +@since version 3.0.0 +*/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id the id of the exception + @param[in] byte_ the byte index where the error occured (or 0 if + the position cannot be determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id, size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id) + "parse error" + + (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character + and n+1 is the index of the terminating null byte or the end of + file. This also holds true when reading a byte vector (CBOR or + MessagePack). + */ + const size_t byte; + + private: + parse_error(int id_, size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) + {} +}; + +/*! +@brief exception indicating errors with iterators + +Exceptions have ids 2xx. + +name / id | example massage | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compated, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id) + what_arg; + return invalid_iterator(id, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; + +/*! +@brief exception indicating executing a member function with a wrong type + +Exceptions have ids 3xx. + +name / id | example massage | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("type_error", id) + what_arg; + return type_error(id, w.c_str()); + } + + private: + type_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; + +/*! +@brief exception indicating access out of the defined range + +Exceptions have ids 4xx. + +name / id | example massage | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id) + what_arg; + return out_of_range(id, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; + +/*! +@brief exception indicating other errors + +Exceptions have ids 5xx. + +name / id | example massage | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id, const std::string& what_arg) + { + std::string w = exception::name("other_error", id) + what_arg; + return other_error(id, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) + : exception(id_, what_arg) + {} +}; + + + +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + @since version 1.0.0 */ -namespace +enum class value_t : uint8_t { + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string +- furthermore, each type is not smaller than itself + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast(lhs)] < + order[static_cast(rhs)]; +} + + +///////////// +// helpers // +///////////// + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +/* +Implementation of two C++17 constructs: conjunction, negation. This is needed +to avoid evaluating all the traits in a condition + +For example: not std::is_same::value and has_value_type::value +will not compile when T = void (on MSVC at least). Whereas +conjunction>, has_value_type>::value will +stop evaluating if negation<...>::value == false + +Please note that those constructs must be used with caution, since symbols can +become very long quickly (which can slow down compilation and cause MSVC +internal compiler errors). Only use it when you have to (see example ahead). +*/ +template struct conjunction : std::true_type {}; +template struct conjunction : B1 {}; +template +struct conjunction : std::conditional, B1>::type {}; + +template struct negation : std::integral_constant < bool, !B::value > {}; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + + +////////////////// +// constructors // +////////////////// + +template struct external_constructor; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_type = value_t::array; + j.m_value = arr; + j.assert_invariant(); + } + + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (bool x : arr) + { + j.m_value.array->push_back(x); + } + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_type = value_t::object; + j.m_value = obj; + j.assert_invariant(); + } + + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.assert_invariant(); + } +}; + + +//////////////////////// +// has_/is_ functions // +//////////////////////// + /*! @brief Helper to determine whether there's a key_type for T. -Thus helper is used to tell associative containers apart from other containers +This helper is used to tell associative containers apart from other containers such as sequence containers. For instance, `std::map` passes the test as it contains a `mapped_type`, whereas `std::vector` fails the test. @sa http://stackoverflow.com/a/7728728/266378 @since version 1.0.0, overworked in version 2.0.6 */ -template -struct has_mapped_type -{ - private: - template - static int detect(U&&); +#define NLOHMANN_JSON_HAS_HELPER(type) \ + template struct has_##type { \ + private: \ + template \ + static int detect(U &&); \ + static void detect(...); \ + public: \ + static constexpr bool value = \ + std::is_integral()))>::value; \ + } - static void detect(...); - public: - static constexpr bool value = - std::is_integral()))>::value; +NLOHMANN_JSON_HAS_HELPER(mapped_type); +NLOHMANN_JSON_HAS_HELPER(key_type); +NLOHMANN_JSON_HAS_HELPER(value_type); +NLOHMANN_JSON_HAS_HELPER(iterator); + +#undef NLOHMANN_JSON_HAS_HELPER + + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl +{ + static constexpr auto value = + std::is_constructible::value and + std::is_constructible::value; }; +template +struct is_compatible_object_type +{ + static auto constexpr value = is_compatible_object_type_impl < + conjunction>, + has_mapped_type, + has_key_type>::value, + typename BasicJsonType::object_t, CompatibleObjectType >::value; +}; + +template +struct is_basic_json_nested_type +{ + static auto constexpr value = std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value; +}; + +template +struct is_compatible_array_type +{ + static auto constexpr value = + conjunction>, + negation>, + negation>, + negation>, + has_value_type, + has_iterator>::value; +}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + std::is_constructible::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type +{ + static constexpr auto value = + is_compatible_integer_type_impl < + std::is_integral::value and + not std::is_same::value, + RealIntegerType, CompatibleNumberIntegerType > ::value; +}; + + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json +{ + private: + // also check the return type of from_json + template::from_json( + std::declval(), std::declval()))>::value>> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json +{ + private: + template < + typename U, + typename = enable_if_t::from_json(std::declval()))>::value >> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +template +struct has_to_json +{ + private: + template::to_json( + std::declval(), std::declval()))> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + + +///////////// +// to_json // +///////////// + +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); } +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template < + typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template < + typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} + +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} + +template < + typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < + is_compatible_array_type::value or + std::is_same::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} + +template < + typename BasicJsonType, typename CompatibleObjectType, + enable_if_t::value, + int> = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& arr) +{ + external_constructor::construct(j, arr); +} + +template ::value, + int> = 0> +void to_json(BasicJsonType& j, T (&arr)[N]) +{ + external_constructor::construct(j, arr); +} + +/////////////// +// from_json // +/////////////// + +// overloads for basic_json template parameters +template::value and + not std::is_same::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast( + *j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (not j.is_boolean()) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + j.type_name())); + } + b = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (not j.is_string()) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + j.type_name())); + } + s = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } + arr = *j.template get_ptr(); +} + +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } + + for (auto it = j.rbegin(), end = j.rend(); it != end; ++it) + { + l.push_front(it->template get()); + } +} + +template +void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>) +{ + using std::begin; + using std::end; + + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} + +template +auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>) +-> decltype( + arr.reserve(std::declval()), + void()) +{ + using std::begin; + using std::end; + + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} + +template::value and + std::is_convertible::value and + not std::is_same::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleArrayType& arr) +{ + if (not j.is_array()) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name())); + } + + from_json_array_impl(j, arr, priority_tag<1> {}); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +{ + if (not j.is_object()) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + j.type_name())); + } + + auto inner_object = j.template get_ptr(); + using std::begin; + using std::end; + // we could avoid the assignment, but this might require a for loop, which + // might be less efficient than the container constructor for some + // containers (would it?) + obj = CompatibleObjectType(begin(*inner_object), end(*inner_object)); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + default: + { + JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name())); + } + } +} + +struct to_json_fn +{ + private: + template + auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } + + template + void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find to_json() method in T's namespace"); + } + + public: + template + void operator()(BasicJsonType& j, T&& val) const + noexcept(noexcept(std::declval().call(j, std::forward(val), priority_tag<1> {}))) + { + return call(j, std::forward(val), priority_tag<1> {}); + } +}; + +struct from_json_fn +{ + private: + template + auto call(const BasicJsonType& j, T& val, priority_tag<1>) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } + + template + void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find from_json() method in T's namespace"); + } + + public: + template + void operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(std::declval().call(j, val, priority_tag<1> {}))) + { + return call(j, val, priority_tag<1> {}); + } +}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail + + +/// namespace to hold default `to_json` / `from_json` functions +namespace +{ +constexpr const auto& to_json = detail::static_const::value; +constexpr const auto& from_json = detail::static_const::value; +} + + +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template + static void from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template + static void to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; + + /*! @brief a class to store JSON values @@ -149,11 +1212,14 @@ default; will be used in @ref number_integer_t) default; will be used in @ref number_float_t) @tparam AllocatorType type of the allocator to use (`std::allocator` by default) +@tparam JSONSerializer the serializer to resolve internal calls to `to_json()` +and `from_json()` (@ref adl_serializer by default) @requirement The class satisfies the following concept requirements: - Basic - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): - JSON values can be default constructed. The result will be a JSON null value. + JSON values can be default constructed. The result will be a JSON null + value. - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): A JSON value can be constructed from an rvalue argument. - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): @@ -168,8 +1234,8 @@ default) - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): JSON values have [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): - All non-static data members are private and standard layout types, the class - has no virtual functions or (virtual) base classes. + All non-static data members are private and standard layout types, the + class has no virtual functions or (virtual) base classes. - Library-wide - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): JSON values can be compared with `==`, see @ref @@ -216,21 +1282,51 @@ template < class NumberIntegerType = std::int64_t, class NumberUnsignedType = std::uint64_t, class NumberFloatType = double, - template class AllocatorType = std::allocator + template class AllocatorType = std::allocator, + template class JSONSerializer = adl_serializer > class basic_json { private: + template friend struct detail::external_constructor; /// workaround type for MSVC using basic_json_t = basic_json; + AllocatorType, JSONSerializer>; public: + using value_t = detail::value_t; // forward declarations template class iter_impl; template class json_reverse_iterator; class json_pointer; + template + using json_serializer = JSONSerializer; + + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + /// @copydoc detail::exception + using exception = detail::exception; + /// @copydoc detail::parse_error + using parse_error = detail::parse_error; + /// @copydoc detail::invalid_iterator + using invalid_iterator = detail::invalid_iterator; + /// @copydoc detail::type_error + using type_error = detail::type_error; + /// @copydoc detail::out_of_range + using out_of_range = detail::out_of_range; + /// @copydoc detail::other_error + using other_error = detail::other_error; + + /// @} + ///////////////////// // container types // @@ -282,6 +1378,81 @@ class basic_json return allocator_type(); } + /*! + @brief returns version information on the library + + This function returns a JSON object with information about the library, + including the version number and information on the platform and compiler. + + @return JSON object holding version information + key | description + ----------- | --------------- + `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). + `copyright` | The copyright line for the library as string. + `name` | The name of the library as string. + `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. + `url` | The URL of the project as string. + `version` | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string). + + @liveexample{The following code shows an example output of the `meta()` + function.,meta} + + @complexity Constant. + + @since 2.1.0 + */ + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2017 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"] = + { + {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1} + }; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + /////////////////////////// // JSON value data types // @@ -449,6 +1620,12 @@ class basic_json std::string @endcode + #### Encoding + + Strings are stored in UTF-8 encoding. Therefore, functions like + `std::string::size()` or `std::string::length()` return the number of + bytes in the string rather than the number of characters or glyphs. + #### String comparison [RFC 7159](http://rfc7159.net/rfc7159) states: @@ -713,47 +1890,6 @@ class basic_json /// @} - - /////////////////////////// - // JSON type enumeration // - /////////////////////////// - - /*! - @brief the JSON type enumeration - - This enumeration collects the different JSON types. It is internally used - to distinguish the stored values, and the functions @ref is_null(), @ref - is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref - is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and - @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and - @ref is_structured() rely on it. - - @note There are three enumeration entries (number_integer, - number_unsigned, and number_float), because the library distinguishes - these three types for numbers: @ref number_unsigned_t is used for unsigned - integers, @ref number_integer_t is used for signed integers, and @ref - number_float_t is used for floating-point numbers or to approximate - integers which do not fit in the limits of their respective type. - - @sa @ref basic_json(const value_t value_type) -- create a JSON value with - the default value for a given type - - @since version 1.0.0 - */ - enum class value_t : uint8_t - { - null, ///< null value - object, ///< object (unordered set of name/value pairs) - array, ///< array (ordered collection of values) - string, ///< string value - boolean, ///< boolean value - number_integer, ///< number value (signed integer) - number_unsigned, ///< number value (unsigned integer) - number_float, ///< number value (floating-point) - discarded ///< discarded by the the parser callback function - }; - - private: /// helper for exception-safe object creation @@ -767,7 +1903,7 @@ class basic_json }; std::unique_ptr object(alloc.allocate(1), deleter); alloc.construct(object.get(), std::forward(args)...); - assert(object.get() != nullptr); + assert(object != nullptr); return object.release(); } @@ -882,7 +2018,7 @@ class basic_json { if (t == value_t::null) { - throw std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.0.10"); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE } break; } @@ -1040,24 +2176,9 @@ class basic_json @complexity Constant. - @throw std::bad_alloc if allocation for object, array, or string value - fails - @liveexample{The following code shows the constructor for different @ref value_t values,basic_json__value_t} - @sa @ref basic_json(std::nullptr_t) -- create a `null` value - @sa @ref basic_json(boolean_t value) -- create a boolean value - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const object_t&) -- create a object value - @sa @ref basic_json(const array_t&) -- create a array value - @sa @ref basic_json(const number_float_t) -- create a number - (floating-point) value - @sa @ref basic_json(const number_integer_t) -- create a number (integer) - value - @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned) - value - @since version 1.0.0 */ basic_json(const value_t value_type) @@ -1091,473 +2212,69 @@ class basic_json } /*! - @brief create an object (explicit) + @brief create a JSON value - Create an object JSON value with a given content. + This is a "catch all" constructor for all compatible JSON types; that is, + types for which a `to_json()` method exsits. The constructor forwards the + parameter @a val to that method (to `json_serializer::to_json` method + with `U = uncvref_t`, to be exact). - @param[in] val a value for the object + Template type @a CompatibleType includes, but is not limited to, the + following types: + - **arrays**: @ref array_t and all kinds of compatible containers such as + `std::vector`, `std::deque`, `std::list`, `std::forward_list`, + `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and + `unordered_multiset` with a `value_type` from which a @ref basic_json + value can be constructed. + - **objects**: @ref object_t and all kinds of compatible associative + containers such as `std::map`, `std::unordered_map`, `std::multimap`, + and `std::unordered_multimap` with a `key_type` compatible to + @ref string_t and a `value_type` from which a @ref basic_json value can + be constructed. + - **strings**: @ref string_t, string literals, and all compatible string + containers can be used. + - **numbers**: @ref number_integer_t, @ref number_unsigned_t, + @ref number_float_t, and all convertible number types such as `int`, + `size_t`, `int64_t`, `float` or `double` can be used. + - **boolean**: @ref boolean_t / `bool` can be used. - @complexity Linear in the size of the passed @a val. + See the examples below. - @throw std::bad_alloc if allocation for object value fails + @tparam CompatibleType a type such that: + - @a CompatibleType is not derived from `std::istream`, + - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move + constructors), + - @a CompatibleType is not a @ref basic_json nested type (e.g., + @ref json_pointer, @ref iterator, etc ...) + - @ref @ref json_serializer has a + `to_json(basic_json_t&, CompatibleType&&)` method - @liveexample{The following code shows the constructor with an @ref - object_t parameter.,basic_json__object_t} + @tparam U = `uncvref_t` - @sa @ref basic_json(const CompatibleObjectType&) -- create an object value - from a compatible STL container + @param[in] val the value to be forwarded - @since version 1.0.0 - */ - basic_json(const object_t& val) - : m_type(value_t::object), m_value(val) - { - assert_invariant(); - } + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. - /*! - @brief create an object (implicit) - - Create an object JSON value with a given content. This constructor allows - any type @a CompatibleObjectType that can be used to construct values of - type @ref object_t. - - @tparam CompatibleObjectType An object type whose `key_type` and - `value_type` is compatible to @ref object_t. Examples include `std::map`, - `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with - a `key_type` of `std::string`, and a `value_type` from which a @ref - basic_json value can be constructed. - - @param[in] val a value for the object - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for object value fails + @throw what `json_serializer::to_json()` throws @liveexample{The following code shows the constructor with several - compatible object type parameters.,basic_json__CompatibleObjectType} + compatible types.,basic_json__CompatibleType} - @sa @ref basic_json(const object_t&) -- create an object value - - @since version 1.0.0 + @since version 2.1.0 */ - template::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleObjectType& val) - : m_type(value_t::object) - { - using std::begin; - using std::end; - m_value.object = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create an array (explicit) - - Create an array JSON value with a given content. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with an @ref array_t - parameter.,basic_json__array_t} - - @sa @ref basic_json(const CompatibleArrayType&) -- create an array value - from a compatible STL containers - - @since version 1.0.0 - */ - basic_json(const array_t& val) - : m_type(value_t::array), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an array (implicit) - - Create an array JSON value with a given content. This constructor allows - any type @a CompatibleArrayType that can be used to construct values of - type @ref array_t. - - @tparam CompatibleArrayType An object type whose `value_type` is - compatible to @ref array_t. Examples include `std::vector`, `std::deque`, - `std::list`, `std::forward_list`, `std::array`, `std::set`, - `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a - `value_type` from which a @ref basic_json value can be constructed. - - @param[in] val a value for the array - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for array value fails - - @liveexample{The following code shows the constructor with several - compatible array type parameters.,basic_json__CompatibleArrayType} - - @sa @ref basic_json(const array_t&) -- create an array value - - @since version 1.0.0 - */ - template::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - not std::is_same::value and - std::is_constructible::value, int>::type = 0> - basic_json(const CompatibleArrayType& val) - : m_type(value_t::array) - { - using std::begin; - using std::end; - m_value.array = create(begin(val), end(val)); - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create an string JSON value with a given content. - - @param[in] val a value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with an @ref - string_t parameter.,basic_json__string_t} - - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const string_t& val) - : m_type(value_t::string), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create a string (explicit) - - Create a string JSON value with a given content. - - @param[in] val a literal value for the string - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the constructor with string literal - parameter.,basic_json__string_t_value_type} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const CompatibleStringType&) -- create a string value - from a compatible string container - - @since version 1.0.0 - */ - basic_json(const typename string_t::value_type* val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a string (implicit) - - Create a string JSON value with a given content. - - @param[in] val a value for the string - - @tparam CompatibleStringType an string type which is compatible to @ref - string_t, for instance `std::string`. - - @complexity Linear in the size of the passed @a val. - - @throw std::bad_alloc if allocation for string value fails - - @liveexample{The following code shows the construction of a string value - from a compatible type.,basic_json__CompatibleStringType} - - @sa @ref basic_json(const string_t&) -- create a string value - @sa @ref basic_json(const typename string_t::value_type*) -- create a - string value from a character pointer - - @since version 1.0.0 - */ - template::value, int>::type = 0> - basic_json(const CompatibleStringType& val) - : basic_json(string_t(val)) - { - assert_invariant(); - } - - /*! - @brief create a boolean (explicit) - - Creates a JSON boolean type from a given value. - - @param[in] val a boolean value to store - - @complexity Constant. - - @liveexample{The example below demonstrates boolean - values.,basic_json__boolean_t} - - @since version 1.0.0 - */ - basic_json(boolean_t val) noexcept - : m_type(value_t::boolean), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number (explicit) - - Create an integer number JSON value with a given content. - - @tparam T A helper type to remove this function via SFINAE in case @ref - number_integer_t is the same as `int`. In this case, this constructor - would have the same signature as @ref basic_json(const int value). Note - the helper type @a T is not visible in this constructor's interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value.,basic_json__number_integer_t} - - @sa @ref basic_json(const int) -- create a number value (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_integer_t val) noexcept - : m_type(value_t::number_integer), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an integer number from an enum type (explicit) - - Create an integer number JSON value with a given content. - - @param[in] val an integer to create a JSON number from - - @note This constructor allows to pass enums directly to a constructor. As - C++ has no way of specifying the type of an anonymous enum explicitly, we - can only rely on the fact that such values implicitly convert to int. As - int may already be the same type of number_integer_t, we may need to - switch off the constructor @ref basic_json(const number_integer_t). - - @complexity Constant. - - @liveexample{The example below shows the construction of an integer - number value from an anonymous enum.,basic_json__const_int} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number - value (integer) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const int val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an integer number (implicit) - - Create an integer number JSON value with a given content. This constructor - allows any type @a CompatibleNumberIntegerType that can be used to - construct values of type @ref number_integer_t. - - @tparam CompatibleNumberIntegerType An integer type which is compatible to - @ref number_integer_t. Examples include the types `int`, `int32_t`, - `long`, and `short`. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @liveexample{The example below shows the construction of several integer - number values from compatible - types.,basic_json__CompatibleIntegerNumberType} - - @sa @ref basic_json(const number_integer_t) -- create a number value - (integer) - @sa @ref basic_json(const int) -- create a number value (integer) - - @since version 1.0.0 - */ - template::value and - std::numeric_limits::is_integer and - std::numeric_limits::is_signed, - CompatibleNumberIntegerType>::type = 0> - basic_json(const CompatibleNumberIntegerType val) noexcept - : m_type(value_t::number_integer), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create an unsigned integer number (explicit) - - Create an unsigned integer number JSON value with a given content. - - @tparam T helper type to compare number_unsigned_t and unsigned int (not - visible in) the interface. - - @param[in] val an integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number - value (unsigned integer) from a compatible number type - - @since version 2.0.0 - */ - template::value) and - std::is_same::value, int>::type = 0> - basic_json(const number_unsigned_t val) noexcept - : m_type(value_t::number_unsigned), m_value(val) - { - assert_invariant(); - } - - /*! - @brief create an unsigned number (implicit) - - Create an unsigned number JSON value with a given content. This - constructor allows any type @a CompatibleNumberUnsignedType that can be - used to construct values of type @ref number_unsigned_t. - - @tparam CompatibleNumberUnsignedType An integer type which is compatible - to @ref number_unsigned_t. Examples may include the types `unsigned int`, - `uint32_t`, or `unsigned short`. - - @param[in] val an unsigned integer to create a JSON number from - - @complexity Constant. - - @sa @ref basic_json(const number_unsigned_t) -- create a number value - (unsigned) - - @since version 2.0.0 - */ - template::value and - std::numeric_limits::is_integer and - not std::numeric_limits::is_signed, - CompatibleNumberUnsignedType>::type = 0> - basic_json(const CompatibleNumberUnsignedType val) noexcept - : m_type(value_t::number_unsigned), - m_value(static_cast(val)) - { - assert_invariant(); - } - - /*! - @brief create a floating-point number (explicit) - - Create a floating-point number JSON value with a given content. - - @param[in] val a floating-point value to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is created - instead. - - @complexity Constant. - - @liveexample{The following example creates several floating-point - values.,basic_json__number_float_t} - - @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number - value (floating-point) from a compatible number type - - @since version 1.0.0 - */ - basic_json(const number_float_t val) noexcept - : m_type(value_t::number_float), m_value(val) - { - // replace infinity and NAN by null - if (not std::isfinite(val)) - { - m_type = value_t::null; - m_value = json_value(); - } - - assert_invariant(); - } - - /*! - @brief create an floating-point number (implicit) - - Create an floating-point number JSON value with a given content. This - constructor allows any type @a CompatibleNumberFloatType that can be used - to construct values of type @ref number_float_t. - - @tparam CompatibleNumberFloatType A floating-point type which is - compatible to @ref number_float_t. Examples may include the types `float` - or `double`. - - @param[in] val a floating-point to create a JSON number from - - @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6 - disallows NaN values: - > Numeric values that cannot be represented in the grammar below (such as - > Infinity and NaN) are not permitted. - In case the parameter @a val is not a number, a JSON null value is - created instead. - - @complexity Constant. - - @liveexample{The example below shows the construction of several - floating-point number values from compatible - types.,basic_json__CompatibleNumberFloatType} - - @sa @ref basic_json(const number_float_t) -- create a number value - (floating-point) - - @since version 1.0.0 - */ - template::value and - std::is_floating_point::value>::type> - basic_json(const CompatibleNumberFloatType val) noexcept - : basic_json(number_float_t(val)) + template, + detail::enable_if_t::value and + not std::is_same::value and + not detail::is_basic_json_nested_type< + basic_json_t, U>::value and + detail::has_to_json::value, + int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer::to_json( + std::declval(), std::forward(val)))) { + JSONSerializer::to_json(*this, std::forward(val)); assert_invariant(); } @@ -1613,10 +2330,12 @@ class basic_json value_t::array and @ref value_t::object are valid); when @a type_deduction is set to `true`, this parameter has no effect - @throw std::domain_error if @a type_deduction is `false`, @a manual_type - is `value_t::object`, but @a init contains an element which is not a pair - whose first element is a string; example: `"cannot create object from - initializer list"` + @throw type_error.301 if @a type_deduction is `false`, @a manual_type is + `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string. In this case, the constructor could not + create an object. If @a type_deduction would have be `true`, an array + would have been created. See @ref object(std::initializer_list) + for an example. @complexity Linear in the size of the initializer list @a init. @@ -1654,7 +2373,7 @@ class basic_json // if object is wanted but impossible, throw an exception if (manual_type == value_t::object and not is_an_object) { - throw std::domain_error("cannot create object from initializer list"); + JSON_THROW(type_error::create(301, "cannot create object from initializer list")); } } @@ -1730,16 +2449,17 @@ class basic_json related function @ref array(std::initializer_list), there are no cases which can only be expressed by this function. That is, any initializer list @a init can also be passed to the initializer list - constructor @ref basic_json(std::initializer_list, bool, - value_t). + constructor @ref basic_json(std::initializer_list, bool, value_t). @param[in] init initializer list to create an object from (optional) @return JSON object value - @throw std::domain_error if @a init is not a pair whose first elements are - strings; thrown by - @ref basic_json(std::initializer_list, bool, value_t) + @throw type_error.301 if @a init is not a list of pairs whose first + elements are strings. In this case, no object can be created. When such a + value is passed to @ref basic_json(std::initializer_list, bool, value_t), + an array would have been created from the passed initializer list @a init. + See example below. @complexity Linear in the size of @a init. @@ -1791,10 +2511,10 @@ class basic_json The semantics depends on the different types a JSON value can have: - In case of primitive types (number, boolean, or string), @a first must be `begin()` and @a last must be `end()`. In this case, the value is - copied. Otherwise, std::out_of_range is thrown. + copied. Otherwise, invalid_iterator.204 is thrown. - In case of structured types (array, object), the constructor behaves as similar versions for `std::vector`. - - In case of a null type, std::domain_error is thrown. + - In case of a null type, invalid_iterator.206 is thrown. @tparam InputIT an input iterator type (@ref iterator or @ref const_iterator) @@ -1805,14 +2525,19 @@ class basic_json @pre Iterators @a first and @a last must be initialized. **This precondition is enforced with an assertion.** - @throw std::domain_error if iterators are not compatible; that is, do not - belong to the same JSON value; example: `"iterators are not compatible"` - @throw std::out_of_range if iterators are for a primitive type (number, - boolean, or string) where an out of range error can be detected easily; - example: `"iterators out of range"` - @throw std::bad_alloc if allocation for object, array, or string fails - @throw std::domain_error if called with a null value; example: `"cannot - use construct with iterators from null"` + @pre Range `[first, last)` is valid. Usually, this precondition cannot be + checked efficiently. Only certain edge cases are detected; see the + description of the exceptions below. + + @throw invalid_iterator.201 if iterators @a first and @a last are not + compatible (i.e., do not belong to the same JSON value). In this case, + the range `[first, last)` is undefined. + @throw invalid_iterator.204 if iterators @a first and @a last belong to a + primitive type (number, boolean, or string), but @a first does not point + to the first element any more. In this case, the range `[first, last)` is + undefined. See example code below. + @throw invalid_iterator.206 if iterators @a first and @a last belong to a + null value. In this case, the range `[first, last)` is undefined. @complexity Linear in distance between @a first and @a last. @@ -1832,7 +2557,7 @@ class basic_json // make sure iterator fits the current value if (first.m_object != last.m_object) { - throw std::domain_error("iterators are not compatible"); + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible")); } // copy type from first iterator @@ -1849,7 +2574,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } break; } @@ -1894,59 +2619,28 @@ class basic_json case value_t::object: { - m_value.object = create(first.m_it.object_iterator, last.m_it.object_iterator); + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); break; } case value_t::array: { - m_value.array = create(first.m_it.array_iterator, last.m_it.array_iterator); + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); break; } default: { - throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()); + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + + first.m_object->type_name())); } } assert_invariant(); } - /*! - @brief construct a JSON value given an input stream - - @param[in,out] i stream to read a serialized JSON value from - @param[in] cb a parser callback function of type @ref parser_callback_t - which is used to control the deserialization by filtering unwanted values - (optional) - - @complexity Linear in the length of the input. The parser is a predictive - LL(1) parser. The complexity can be higher if the parser callback function - @a cb has a super-linear complexity. - - @note A UTF-8 byte order mark is silently ignored. - - @deprecated This constructor is deprecated and will be removed in version - 3.0.0 to unify the interface of the library. Deserialization will be - done by stream operators or by calling one of the `parse` functions, - e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls - like `json j(i);` for an input stream @a i need to be replaced by - `json j = json::parse(i);`. See the example below. - - @liveexample{The example below demonstrates constructing a JSON value from - a `std::stringstream` with and without callback - function.,basic_json__istream} - - @since version 2.0.0, deprecated in version 2.0.3, to be removed in - version 3.0.0 - */ - JSON_DEPRECATED - explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) - { - *this = parser(i, cb).parse(); - assert_invariant(); - } /////////////////////////////////////// // other constructors and destructor // @@ -1967,8 +2661,6 @@ class basic_json - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. - @throw std::bad_alloc if allocation for object, array, or string fails. - @liveexample{The following code shows an example for the copy constructor.,basic_json__basic_json} @@ -2196,22 +2888,15 @@ class basic_json string_t dump(const int indent = -1) const { std::stringstream ss; - // fix locale problems - ss.imbue(std::locale::classic()); - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - ss.precision(std::numeric_limits::digits10); + serializer s(ss); if (indent >= 0) { - dump(ss, true, static_cast(indent)); + s.dump(*this, true, static_cast(indent)); } else { - dump(ss, false, 0); + s.dump(*this, false, 0); } return ss.str(); @@ -2579,244 +3264,97 @@ class basic_json // value access // ////////////////// - /// get an object (explicit) - template::value and - std::is_convertible::value, int>::type = 0> - T get_impl(T*) const - { - if (is_object()) - { - return T(m_value.object->begin(), m_value.object->end()); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an object (explicit) - object_t get_impl(object_t*) const - { - if (is_object()) - { - return *(m_value.object); - } - else - { - throw std::domain_error("type must be object, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value and - not std::is_arithmetic::value and - not std::is_convertible::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - T to_vector; - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not std::is_same::value, int>::type = 0> - std::vector get_impl(std::vector*) const - { - if (is_array()) - { - std::vector to_vector; - to_vector.reserve(m_value.array->size()); - std::transform(m_value.array->begin(), m_value.array->end(), - std::inserter(to_vector, to_vector.end()), [](basic_json i) - { - return i.get(); - }); - return to_vector; - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - template::value and - not has_mapped_type::value, int>::type = 0> - T get_impl(T*) const - { - if (is_array()) - { - return T(m_value.array->begin(), m_value.array->end()); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get an array (explicit) - array_t get_impl(array_t*) const - { - if (is_array()) - { - return *(m_value.array); - } - else - { - throw std::domain_error("type must be array, but is " + type_name()); - } - } - - /// get a string (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - if (is_string()) - { - return *m_value.string; - } - else - { - throw std::domain_error("type must be string, but is " + type_name()); - } - } - - /// get a number (explicit) - template::value, int>::type = 0> - T get_impl(T*) const - { - switch (m_type) - { - case value_t::number_integer: - { - return static_cast(m_value.number_integer); - } - - case value_t::number_unsigned: - { - return static_cast(m_value.number_unsigned); - } - - case value_t::number_float: - { - return static_cast(m_value.number_float); - } - - default: - { - throw std::domain_error("type must be number, but is " + type_name()); - } - } - } - /// get a boolean (explicit) - constexpr boolean_t get_impl(boolean_t*) const + boolean_t get_impl(boolean_t* /*unused*/) const { - return is_boolean() - ? m_value.boolean - : throw std::domain_error("type must be boolean, but is " + type_name()); + if (is_boolean()) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + type_name())); } /// get a pointer to the value (object) - object_t* get_impl_ptr(object_t*) noexcept + object_t* get_impl_ptr(object_t* /*unused*/) noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (object) - constexpr const object_t* get_impl_ptr(const object_t*) const noexcept + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept { return is_object() ? m_value.object : nullptr; } /// get a pointer to the value (array) - array_t* get_impl_ptr(array_t*) noexcept + array_t* get_impl_ptr(array_t* /*unused*/) noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (array) - constexpr const array_t* get_impl_ptr(const array_t*) const noexcept + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept { return is_array() ? m_value.array : nullptr; } /// get a pointer to the value (string) - string_t* get_impl_ptr(string_t*) noexcept + string_t* get_impl_ptr(string_t* /*unused*/) noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (string) - constexpr const string_t* get_impl_ptr(const string_t*) const noexcept + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept { return is_string() ? m_value.string : nullptr; } /// get a pointer to the value (boolean) - boolean_t* get_impl_ptr(boolean_t*) noexcept + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (boolean) - constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept { return is_boolean() ? &m_value.boolean : nullptr; } /// get a pointer to the value (integer number) - number_integer_t* get_impl_ptr(number_integer_t*) noexcept + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (integer number) - constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept { return is_number_integer() ? &m_value.number_integer : nullptr; } /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept { return is_number_unsigned() ? &m_value.number_unsigned : nullptr; } /// get a pointer to the value (floating-point number) - number_float_t* get_impl_ptr(number_float_t*) noexcept + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept { return is_number_float() ? &m_value.number_float : nullptr; } /// get a pointer to the value (floating-point number) - constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept { return is_number_float() ? &m_value.number_float : nullptr; } @@ -2829,7 +3367,7 @@ class basic_json @tparam ThisType will be deduced as `basic_json` or `const basic_json` - @throw std::domain_error if ReferenceType does not match underlying value + @throw type_error.303 if ReferenceType does not match underlying value type of the current JSON */ template @@ -2845,34 +3383,68 @@ class basic_json { return *ptr; } - else - { - throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " + - obj.type_name()); - } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + obj.type_name())); } public: - /// @name value access /// Direct access to the stored value of a JSON value. /// @{ + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template < + typename BasicJsonType, + detail::enable_if_t::type, + basic_json_t>::value, + int> = 0 > + basic_json get() const + { + return *this; + } + /*! @brief get a value (explicit) - Explicit type conversion between the JSON value and a compatible value. + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. - @tparam ValueType non-pointer type compatible to the JSON value, for - instance `int` for JSON integer numbers, `bool` for JSON booleans, or - `std::vector` types for JSON arrays + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode - @return copy of the JSON value, converted to type @a ValueType + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const @ref basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const @ref basic_json&)` - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON; example: `"type must be object, but is null"` + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type - @complexity Linear in the size of the JSON value. + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws @liveexample{The example below shows several conversions from JSON values to other types. There a few things to note: (1) Floating-point numbers can @@ -2881,21 +3453,75 @@ class basic_json associative containers such as `std::unordered_map`.,get__ValueType_const} - @internal - The idea of using a casted null pointer to choose the correct - implementation is from . - @endinternal - - @sa @ref operator ValueType() const for implicit conversion - @sa @ref get() for pointer-member access - - @since version 1.0.0 + @since version 2.1.0 */ - template::value, int>::type = 0> - ValueType get() const + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t < + not std::is_same::value and + detail::has_from_json::value and + not detail::has_non_default_from_json::value, + int > = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) { - return get_impl(static_cast(nullptr)); + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + static_assert(std::is_default_constructible::value, + "types must be DefaultConstructible when used with get()"); + + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const @ref basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 + */ + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t::value and + detail::has_non_default_from_json::value, int> = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) + { + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return JSONSerializer::from_json(*this); } /*! @@ -3025,7 +3651,7 @@ class basic_json /*! @brief get a reference value (implicit) - Implict reference access to the internally stored JSON value. No copies + Implicit reference access to the internally stored JSON value. No copies are made. @warning Writing data to the referee of the result yields an undefined @@ -3037,10 +3663,10 @@ class basic_json @return reference to the internally stored JSON value if the requested reference type @a ReferenceType fits to the JSON value; throws - std::domain_error otherwise + type_error.303 otherwise - @throw std::domain_error in case passed type @a ReferenceType is - incompatible with the stored JSON value + @throw type_error.303 in case passed type @a ReferenceType is incompatible + with the stored JSON value; see example below @complexity Constant. @@ -3083,8 +3709,9 @@ class basic_json @return copy of the JSON value, converted to type @a ValueType - @throw std::domain_error in case passed type @a ValueType is incompatible - to JSON, thrown by @ref get() const + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below @complexity Linear in the size of the JSON value. @@ -3100,8 +3727,11 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer::value and not std::is_same::value -#ifndef _MSC_VER // Fix for issue #167 operator<< abiguity under VS2015 +#ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same>::value +#endif +#if defined(_MSC_VER) && _MSC_VER >1900 && defined(_HAS_CXX17) && _HAS_CXX17 == 1 // fix for issue #464 + and not std::is_same::value #endif , int >::type = 0 > operator ValueType() const @@ -3131,36 +3761,40 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. @complexity Constant. - @liveexample{The example below shows how array elements can be read and - written using `at()`.,at__size_type} - @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__size_type} */ reference at(size_type idx) { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3174,36 +3808,40 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if the JSON value is not an array; example: - `"cannot use at() with string"` - @throw std::out_of_range if the index @a idx is out of range of the array; - that is, `idx >= size()`; example: `"array index 7 is out of range"` + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. @complexity Constant. - @liveexample{The example below shows how array elements can be read using - `at()`.,at__size_type_const} - @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__size_type_const} */ const_reference at(size_type idx) const { // at only works for arrays if (is_array()) { - try + JSON_TRY { return m_value.array->at(idx); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3217,40 +3855,44 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. @complexity Logarithmic in the size of the container. - @liveexample{The example below shows how object elements can be read and - written using `at()`.,at__object_t_key_type} - @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__object_t_key_type} */ reference at(const typename object_t::key_type& key) { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3264,40 +3906,44 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if the JSON value is not an object; example: - `"cannot use at() with boolean"` - @throw std::out_of_range if the key @a key is is not stored in the object; - that is, `find(key) == end()`; example: `"key "the fast" not found"` + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. @complexity Logarithmic in the size of the container. - @liveexample{The example below shows how object elements can be read using - `at()`.,at__object_t_key_type_const} - @sa @ref operator[](const typename object_t::key_type&) for unchecked access by reference @sa @ref value() for access by value with a default value @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__object_t_key_type_const} */ const_reference at(const typename object_t::key_type& key) const { // at only works for objects if (is_object()) { - try + JSON_TRY { return m_value.object->at(key); } - catch (std::out_of_range&) + JSON_CATCH (std::out_of_range&) { // create better exception explanation - throw std::out_of_range("key '" + key + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); } } else { - throw std::domain_error("cannot use at() with " + type_name()); + JSON_THROW(type_error::create(304, "cannot use at() with " + type_name())); } } @@ -3314,8 +3960,8 @@ class basic_json @return reference to the element at index @a idx - @throw std::domain_error if JSON is not an array or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an array or null; in that + cases, using the [] operator with an index makes no sense. @complexity Constant if @a idx is in the range of the array. Otherwise linear in `idx - size()`. @@ -3349,10 +3995,8 @@ class basic_json return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3364,8 +4008,8 @@ class basic_json @return const reference to the element at index @a idx - @throw std::domain_error if JSON is not an array; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an array; in that cases, + using the [] operator with an index makes no sense. @complexity Constant. @@ -3381,10 +4025,8 @@ class basic_json { return m_value.array->operator[](idx); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3400,8 +4042,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3429,10 +4071,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3451,8 +4091,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3473,10 +4113,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3492,8 +4130,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3527,8 +4165,8 @@ class basic_json @return const reference to the element at key @a key - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3560,8 +4198,8 @@ class basic_json @return reference to the element at key @a key - @throw std::domain_error if JSON is not an object or null; example: - `"cannot use operator[] with string"` + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3590,10 +4228,8 @@ class basic_json { return m_value.object->operator[](key); } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3612,8 +4248,8 @@ class basic_json @pre The element with key @a key must exist. **This precondition is enforced with an assertion.** - @throw std::domain_error if JSON is not an object; example: `"cannot use - operator[] with null"` + @throw type_error.305 if the JSON value is not an object; in that cases, + using the [] operator with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3635,10 +4271,8 @@ class basic_json assert(m_value.object->find(key) != m_value.object->end()); return m_value.object->find(key)->second; } - else - { - throw std::domain_error("cannot use operator[] with " + type_name()); - } + + JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name())); } /*! @@ -3651,7 +4285,7 @@ class basic_json @code {.cpp} try { return at(key); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3674,8 +4308,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3702,14 +4336,12 @@ class basic_json { return *it; } - else - { - return default_value; - } + + return default_value; } else { - throw std::domain_error("cannot use value() with " + type_name()); + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } } @@ -3732,7 +4364,7 @@ class basic_json @code {.cpp} try { return at(ptr); - } catch(std::out_of_range) { + } catch(out_of_range) { return default_value; } @endcode @@ -3751,8 +4383,8 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw std::domain_error if JSON is not an object; example: `"cannot use - value() with null"` + @throw type_error.306 if the JSON value is not an objec; in that cases, + using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -3771,19 +4403,17 @@ class basic_json if (is_object()) { // if pointer resolves a value, return it or use default value - try + JSON_TRY { return ptr.get_checked(this); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { return default_value; } } - else - { - throw std::domain_error("cannot use value() with " + type_name()); - } + + JSON_THROW(type_error::create(306, "cannot use value() with " + type_name())); } /*! @@ -3812,7 +4442,7 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value + @throw invalid_iterator.214 when called on `null` value @liveexample{The following code shows an example for `front()`.,front} @@ -3855,7 +4485,8 @@ class basic_json assertions**). @post The JSON value remains unchanged. - @throw std::out_of_range when called on `null` value. + @throw invalid_iterator.214 when called on a `null` value. See example + below. @liveexample{The following code shows an example for `back()`.,back} @@ -3899,17 +4530,18 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on an iterator which does not belong to - the current JSON value; example: `"iterator does not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.202 if called on an iterator which does not belong + to the current JSON value; example: `"iterator does not fit current + value"` + @throw invalid_iterator.205 if called on a primitive type with invalid iterator (i.e., any iterator which is not `begin()`); example: `"iterator out of range"` @complexity The complexity depends on the type: - objects: amortized constant - - arrays: linear in distance between pos and the end of the container + - arrays: linear in distance between @a pos and the end of the container - strings: linear in the length of the string - other types: constant @@ -3934,7 +4566,7 @@ class basic_json // make sure iterator fits the current value if (this != pos.m_object) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } IteratorType result = end(); @@ -3949,7 +4581,7 @@ class basic_json { if (not pos.m_it.primitive_iterator.is_begin()) { - throw std::out_of_range("iterator out of range"); + JSON_THROW(invalid_iterator::create(205, "iterator out of range")); } if (is_string()) @@ -3979,7 +4611,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4006,11 +4638,11 @@ class basic_json @post Invalidates iterators and references at or after the point of the erase, including the `end()` iterator. - @throw std::domain_error if called on a `null` value; example: `"cannot - use erase() with null"` - @throw std::domain_error if called on iterators which does not belong to - the current JSON value; example: `"iterators do not fit current value"` - @throw std::out_of_range if called on a primitive type with invalid + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.203 if called on iterators which does not belong + to the current JSON value; example: `"iterators do not fit current value"` + @throw invalid_iterator.204 if called on a primitive type with invalid iterators (i.e., if `first != begin()` and `last != end()`); example: `"iterators out of range"` @@ -4041,7 +4673,7 @@ class basic_json // make sure iterator fits the current value if (this != first.m_object or this != last.m_object) { - throw std::domain_error("iterators do not fit current value"); + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value")); } IteratorType result = end(); @@ -4056,7 +4688,7 @@ class basic_json { if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) { - throw std::out_of_range("iterators out of range"); + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); } if (is_string()) @@ -4088,7 +4720,7 @@ class basic_json default: { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4109,7 +4741,7 @@ class basic_json @post References and iterators to the erased elements are invalidated. Other references and iterators are not affected. - @throw std::domain_error when called on a type other than JSON object; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` @complexity `log(size()) + count(key)` @@ -4131,10 +4763,8 @@ class basic_json { return m_value.object->erase(key); } - else - { - throw std::domain_error("cannot use erase() with " + type_name()); - } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } /*! @@ -4144,9 +4774,9 @@ class basic_json @param[in] idx index of the element to remove - @throw std::domain_error when called on a type other than JSON array; + @throw type_error.307 when called on a type other than JSON object; example: `"cannot use erase() with null"` - @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + @throw out_of_range.401 when `idx >= size()`; example: `"array index 17 is out of range"` @complexity Linear in distance between @a idx and the end of the container. @@ -4168,14 +4798,14 @@ class basic_json { if (idx >= size()) { - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } m_value.array->erase(m_value.array->begin() + static_cast(idx)); } else { - throw std::domain_error("cannot use erase() with " + type_name()); + JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name())); } } @@ -4877,7 +5507,7 @@ class basic_json @param[in] val the value to add to the JSON array - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.308 when called on a type other than JSON array or null; example: `"cannot use push_back() with number"` @complexity Amortized constant. @@ -4893,7 +5523,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4929,7 +5559,7 @@ class basic_json // push_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an array @@ -4963,7 +5593,7 @@ class basic_json @param[in] val the value to add to the JSON object - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.308 when called on a type other than JSON object or null; example: `"cannot use push_back() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -4979,7 +5609,7 @@ class basic_json // push_back only works for null objects or objects if (not(is_null() or is_object())) { - throw std::domain_error("cannot use push_back() with " + type_name()); + JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name())); } // transform null object into an object @@ -5017,7 +5647,7 @@ class basic_json @ref push_back(const typename object_t::value_type&). Otherwise, @a init is converted to a JSON value and added using @ref push_back(basic_json&&). - @param init an initializer list + @param[in] init an initializer list @complexity Linear in the size of the initializer list @a init. @@ -5062,7 +5692,7 @@ class basic_json @param[in] args arguments to forward to a constructor of @ref basic_json @tparam Args compatible types to create a @ref basic_json object - @throw std::domain_error when called on a type other than JSON array or + @throw type_error.311 when called on a type other than JSON array or null; example: `"cannot use emplace_back() with number"` @complexity Amortized constant. @@ -5079,7 +5709,7 @@ class basic_json // emplace_back only works for null objects or arrays if (not(is_null() or is_array())) { - throw std::domain_error("cannot use emplace_back() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + type_name())); } // transform null object into an array @@ -5097,8 +5727,8 @@ class basic_json /*! @brief add an object to an object if key does not exist - Inserts a new element into a JSON object constructed in-place with the given - @a args if there is no element with the key in the container. If the + Inserts a new element into a JSON object constructed in-place with the + given @a args if there is no element with the key in the container. If the function is called on a JSON null value, an empty object is created before appending the value created from @a args. @@ -5109,7 +5739,7 @@ class basic_json already-existing element if no insertion happened, and a bool denoting whether the insertion took place. - @throw std::domain_error when called on a type other than JSON object or + @throw type_error.311 when called on a type other than JSON object or null; example: `"cannot use emplace() with number"` @complexity Logarithmic in the size of the container, O(log(`size()`)). @@ -5127,7 +5757,7 @@ class basic_json // emplace only works for null objects or arrays if (not(is_null() or is_object())) { - throw std::domain_error("cannot use emplace() with " + type_name()); + JSON_THROW(type_error::create(311, "cannot use emplace() with " + type_name())); } // transform null object into an object @@ -5158,13 +5788,13 @@ class basic_json @param[in] val element to insert @return iterator pointing to the inserted @a val. - @throw std::domain_error if called on JSON values other than arrays; + @throw type_error.309 if called on JSON values other than arrays; example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` - @complexity Constant plus linear in the distance between pos and end of the - container. + @complexity Constant plus linear in the distance between @a pos and end of + the container. @liveexample{The example shows how `insert()` is used.,insert} @@ -5178,7 +5808,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5186,10 +5816,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5213,10 +5841,10 @@ class basic_json @return iterator pointing to the first element inserted, or @a pos if `cnt==0` - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @complexity Linear in @a cnt plus linear in the distance between @a pos and end of the container. @@ -5233,7 +5861,7 @@ class basic_json // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5241,10 +5869,8 @@ class basic_json result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); return result; } - else - { - throw std::domain_error("cannot use insert() with " + type_name()); - } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } /*! @@ -5257,13 +5883,13 @@ class basic_json @param[in] first begin of the range of elements to insert @param[in] last end of the range of elements to insert - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` - @throw std::domain_error if @a first and @a last do not belong to the same - JSON value; example: `"iterators do not fit"` - @throw std::domain_error if @a first or @a last are iterators into + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + @throw invalid_iterator.211 if @a first or @a last are iterators into container for which insert is called; example: `"passed iterators may not belong to container"` @@ -5282,24 +5908,24 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // check if range iterators belong to the same JSON object if (first.m_object != last.m_object) { - throw std::domain_error("iterators do not fit"); + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); } if (first.m_object == this or last.m_object == this) { - throw std::domain_error("passed iterators may not belong to container"); + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container")); } // insert to array and return iterator @@ -5320,10 +5946,10 @@ class basic_json the end() iterator @param[in] ilist initializer list to insert the values from - @throw std::domain_error if called on JSON values other than arrays; - example: `"cannot use insert() with string"` - @throw std::domain_error if @a pos is not an iterator of *this; example: - `"iterator does not fit current value"` + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` @return iterator pointing to the first element inserted, or @a pos if `ilist` is empty @@ -5340,13 +5966,13 @@ class basic_json // insert only works for arrays if (not is_array()) { - throw std::domain_error("cannot use insert() with " + type_name()); + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); } // check if iterator pos fits to this JSON value if (pos.m_object != this) { - throw std::domain_error("iterator does not fit current value"); + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); } // insert to array and return iterator @@ -5355,6 +5981,52 @@ class basic_json return result; } + /*! + @brief inserts elements + + Inserts elements from range `[first, last)`. + + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.309 if called on JSON values other than objects; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if iterator @a first or @a last does does not + point to an object; example: `"iterators first and last must point to + objects"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + + @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number + of elements to insert. + + @liveexample{The example shows how `insert()` is used.,insert__range_object} + + @since version 3.0.0 + */ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (not is_object()) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name())); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + // passed iterators must belong to objects + if (not first.m_object->is_object() or not first.m_object->is_object()) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + /*! @brief exchanges the values @@ -5394,7 +6066,7 @@ class basic_json @param[in,out] other array to exchange the contents with - @throw std::domain_error when JSON value is not an array; example: `"cannot + @throw type_error.310 when JSON value is not an array; example: `"cannot use swap() with string"` @complexity Constant. @@ -5413,7 +6085,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5427,7 +6099,7 @@ class basic_json @param[in,out] other object to exchange the contents with - @throw std::domain_error when JSON value is not an object; example: + @throw type_error.310 when JSON value is not an object; example: `"cannot use swap() with string"` @complexity Constant. @@ -5446,7 +6118,7 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } @@ -5460,7 +6132,7 @@ class basic_json @param[in,out] other string to exchange the contents with - @throw std::domain_error when JSON value is not a string; example: `"cannot + @throw type_error.310 when JSON value is not a string; example: `"cannot use swap() with boolean"` @complexity Constant. @@ -5479,13 +6151,13 @@ class basic_json } else { - throw std::domain_error("cannot use swap() with " + type_name()); + JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name())); } } /// @} - + public: ////////////////////////////////////////// // lexicographical comparison operators // ////////////////////////////////////////// @@ -5493,52 +6165,22 @@ class basic_json /// @name lexicographical comparison operators /// @{ - private: - /*! - @brief comparison operator for JSON types - - Returns an ordering that is similar to Python: - - order: null < boolean < number < object < array < string - - furthermore, each type is not smaller than itself - - @since version 1.0.0 - */ - friend bool operator<(const value_t lhs, const value_t rhs) noexcept - { - static constexpr std::array order = {{ - 0, // null - 3, // object - 4, // array - 5, // string - 1, // boolean - 2, // integer - 2, // unsigned - 2, // float - } - }; - - // discarded values are not comparable - if (lhs == value_t::discarded or rhs == value_t::discarded) - { - return false; - } - - return order[static_cast(lhs)] < order[static_cast(rhs)]; - } - - public: /*! @brief comparison: equal Compares two JSON values for equality according to the following rules: - Two JSON values are equal if (1) they are from the same type and (2) - their stored values are the same. + their stored values are the same according to their respective + `operator==`. - Integer and floating-point numbers are automatically converted before comparison. Floating-point numbers are compared indirectly: two floating-point numbers `f1` and `f2` are considered equal if neither - `f1 > f2` nor `f2 > f1` holds. + `f1 > f2` nor `f2 > f1` holds. Note than two NaN values are always + treated as unequal. - Two JSON null values are equal. + @note NaN values never compare equal to themselves or to other NaN values. + @param[in] lhs first JSON value to consider @param[in] rhs second JSON value to consider @return whether the values @a lhs and @a rhs are equal @@ -5627,34 +6269,24 @@ class basic_json /*! @brief comparison: equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__equal__nullptr_t} - - @since version 1.0.0 + @copydoc operator==(const_reference, const_reference) */ - friend bool operator==(const_reference v, std::nullptr_t) noexcept + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept { - return v.is_null(); + return (lhs == basic_json(rhs)); } /*! @brief comparison: equal - @copydoc operator==(const_reference, std::nullptr_t) + @copydoc operator==(const_reference, const_reference) */ - friend bool operator==(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept { - return v.is_null(); + return (basic_json(lhs) == rhs); } /*! @@ -5680,34 +6312,24 @@ class basic_json /*! @brief comparison: not equal - - The functions compares the given JSON value against a null pointer. As the - null pointer can be used to initialize a JSON value to null, a comparison - of JSON value @a v with a null pointer should be equivalent to call - `not v.is_null()`. - - @param[in] v JSON value to consider - @return whether @a v is not null - - @complexity Constant. - - @liveexample{The example compares several JSON types to the null pointer. - ,operator__notequal__nullptr_t} - - @since version 1.0.0 + @copydoc operator!=(const_reference, const_reference) */ - friend bool operator!=(const_reference v, std::nullptr_t) noexcept + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept { - return not v.is_null(); + return (lhs != basic_json(rhs)); } /*! @brief comparison: not equal - @copydoc operator!=(const_reference, std::nullptr_t) + @copydoc operator!=(const_reference, const_reference) */ - friend bool operator!=(std::nullptr_t, const_reference v) noexcept + template::value, int>::type = 0> + friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept { - return not v.is_null(); + return (basic_json(lhs) != rhs); } /*! @@ -5812,6 +6434,28 @@ class basic_json return operator<(lhs_type, rhs_type); } + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs < basic_json(rhs)); + } + + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) < rhs); + } + /*! @brief comparison: less than or equal @@ -5834,6 +6478,28 @@ class basic_json return not (rhs < lhs); } + /*! + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs <= basic_json(rhs)); + } + + /*! + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) <= rhs); + } + /*! @brief comparison: greater than @@ -5856,6 +6522,28 @@ class basic_json return not (lhs <= rhs); } + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs > basic_json(rhs)); + } + + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) > rhs); + } + /*! @brief comparison: greater than or equal @@ -5878,6 +6566,28 @@ class basic_json return not (lhs < rhs); } + /*! + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs >= basic_json(rhs)); + } + + /*! + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) >= rhs); + } + /// @} @@ -5888,6 +6598,593 @@ class basic_json /// @name serialization /// @{ + private: + /*! + @brief wrapper around the serialization functions + */ + class serializer + { + private: + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + + public: + /*! + @param[in] s output stream to serialize to + */ + serializer(std::ostream& s) + : o(s), loc(std::localeconv()), + thousands_sep(!loc->thousands_sep ? '\0' : loc->thousands_sep[0]), + decimal_point(!loc->decimal_point ? '\0' : loc->decimal_point[0]) + {} + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and + organizes the serialization internally. The indentation level is + propagated as additional parameter. In case of arrays and objects, the + function is called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const basic_json& val, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o.write("{}", 2); + return; + } + + if (pretty_print) + { + o.write("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(i != val.m_value.object->cend()); + o.write(indent_string.c_str(), static_cast(new_indent)); + o.put('\"'); + dump_escaped(i->first); + o.write("\": ", 3); + dump(i->second, true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put('}'); + } + else + { + o.put('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(i != val.m_value.object->cend()); + o.put('\"'); + dump_escaped(i->first); + o.write("\":", 2); + dump(i->second, false, indent_step, current_indent); + + o.put('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o.write("[]", 2); + return; + } + + if (pretty_print) + { + o.write("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (indent_string.size() < new_indent) + { + indent_string.resize(new_indent, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(*i, true, indent_step, new_indent); + o.write(",\n", 2); + } + + // last element + assert(not val.m_value.array->empty()); + o.write(indent_string.c_str(), static_cast(new_indent)); + dump(val.m_value.array->back(), true, indent_step, new_indent); + + o.put('\n'); + o.write(indent_string.c_str(), static_cast(current_indent)); + o.put(']'); + } + else + { + o.put('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, indent_step, current_indent); + o.put(','); + } + + // last element + assert(not val.m_value.array->empty()); + dump(val.m_value.array->back(), false, indent_step, current_indent); + + o.put(']'); + } + + return; + } + + case value_t::string: + { + o.put('\"'); + dump_escaped(*val.m_value.string); + o.put('\"'); + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o.write("true", 4); + } + else + { + o.write("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o.write("", 11); + return; + } + + case value_t::null: + { + o.write("null", 4); + return; + } + } + } + + private: + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + + default: + { + return res; + } + } + }); + } + + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence + of an escape character (backslash) and another character and other + control characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s) const + { + const auto space = extra_space(s); + if (space == 0) + { + o.write(s.c_str(), static_cast(s.size())); + return; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x0b: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1a: + case 0x1b: + case 0x1c: + case 0x1d: + case 0x1e: + case 0x1f: + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + break; + } + + default: + { + // all other characters are added as-is + result[pos++] = c; + break; + } + } + } + + assert(pos == s.size() + space); + o.write(result.c_str(), static_cast(result.size())); + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template::value or + std::is_same::value, int> = 0> + void dump_integer(NumberType x) + { + // special case for "0" + if (x == 0) + { + o.put('0'); + return; + } + + const bool is_negative = x < 0; + size_t i = 0; + + // spare 1 byte for '\0' + while (x != 0 and i < number_buffer.size() - 1) + { + const auto digit = std::labs(static_cast(x % 10)); + number_buffer[i++] = static_cast('0' + digit); + x /= 10; + } + + // make sure the number has been processed completely + assert(x == 0); + + if (is_negative) + { + // make sure there is capacity for the '-' + assert(i < number_buffer.size() - 2); + number_buffer[i++] = '-'; + } + + std::reverse(number_buffer.begin(), number_buffer.begin() + i); + o.write(number_buffer.data(), static_cast(i)); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works + internally with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (not std::isfinite(x) or std::isnan(x)) + { + o.write("null", 4); + return; + } + + // special case for 0.0 and -0.0 + if (x == 0) + { + if (std::signbit(x)) + { + o.write("-0.0", 4); + } + else + { + o.write("0.0", 3); + } + return; + } + + // get number of digits for a text -> float -> text round-trip + static constexpr auto d = std::numeric_limits::digits10; + + // the actual conversion + std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), + "%.*g", d, x); + + // negative value indicates an error + assert(len > 0); + // check if buffer was large enough + assert(static_cast(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(number_buffer.begin(), + number_buffer.begin() + len, + thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + assert((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + for (auto& c : number_buffer) + { + if (c == decimal_point) + { + c = '.'; + break; + } + } + } + + o.write(number_buffer.data(), static_cast(len)); + + // determine if need to append ".0" + const bool value_is_int_like = std::none_of(number_buffer.begin(), + number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' or c == 'e'; + }); + + if (value_is_int_like) + { + o.write(".0", 2); + } + } + + private: + /// the output of the serializer + std::ostream& o; + + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// the indentation string + string_t indent_string = string_t(512, ' '); + }; + + public: /*! @brief serialize to stream @@ -5898,10 +7195,6 @@ class basic_json `std::setw(4)` on @a o sets the indentation level to `4` and the serialization result is the same as calling `dump(4)`. - @note During serializaion, the locale and the precision of the output - stream @a o are changed. The original values are restored when the - function returns. - @param[in,out] o stream to serialize to @param[in] j JSON value to serialize @@ -5923,29 +7216,20 @@ class basic_json // reset width to 0 for subsequent calls to this stream o.width(0); - // fix locale problems - const auto old_locale = o.imbue(std::locale::classic()); - // set precision - - // 6, 15 or 16 digits of precision allows round-trip IEEE 754 - // string->float->string, string->double->string or string->long - // double->string; to be safe, we read this value from - // std::numeric_limits::digits10 - const auto old_precision = o.precision(std::numeric_limits::digits10); - // do the actual serialization - j.dump(o, pretty_print, static_cast(indentation)); - - // reset locale and precision - o.imbue(old_locale); - o.precision(old_precision); + serializer s(o); + s.dump(j, pretty_print, static_cast(indentation)); return o; } /*! @brief serialize to stream - @copydoc operator<<(std::ostream&, const basic_json&) + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::ostream& operator<<(std::ostream&, const basic_json&) + instead; that is, replace calls like `j >> o;` with `o << j;`. */ + JSON_DEPRECATED friend std::ostream& operator>>(const basic_json& j, std::ostream& o) { return o << j; @@ -5977,6 +7261,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6007,6 +7296,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6043,6 +7336,11 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6102,6 +7400,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6122,7 +7424,7 @@ class basic_json { // assertion to check that the iterator range is indeed contiguous, // see http://stackoverflow.com/a/35008842/266378 for more discussion - assert(std::accumulate(first, last, std::make_pair(true, 0), + assert(std::accumulate(first, last, std::pair(true, 0), [&first](std::pair res, decltype(*first) val) { res.first &= (val == *(std::next(std::addressof(*first), res.second++))); @@ -6172,6 +7474,10 @@ class basic_json @return result of the deserialization + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. The complexity can be higher if the parser callback function @a cb has a super-linear complexity. @@ -6196,6 +7502,20 @@ class basic_json return parse(std::begin(c), std::end(c), cb); } + /*! + @brief deserialize from stream + @deprecated This stream operator is deprecated and will be removed in a + future version of the library. Please use + @ref std::istream& operator>>(std::istream&, basic_json&) + instead; that is, replace calls like `j << i;` with `i >> j;`. + */ + JSON_DEPRECATED + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + /*! @brief deserialize from stream @@ -6204,7 +7524,10 @@ class basic_json @param[in,out] i input stream to read a serialized JSON value from @param[in,out] j JSON value to write the deserialized input to - @throw std::invalid_argument in case of parse errors + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + @throw parse_error.111 if input stream is in a bad state @complexity Linear in the length of the input. The parser is a predictive LL(1) parser. @@ -6219,16 +7542,6 @@ class basic_json @since version 1.0.0 */ - friend std::istream& operator<<(basic_json& j, std::istream& i) - { - j = parser(i).parse(); - return i; - } - - /*! - @brief deserialize from stream - @copydoc operator<<(basic_json&, std::istream&) - */ friend std::istream& operator>>(std::istream& i, basic_json& j) { j = parser(i).parse(); @@ -6245,6 +7558,11 @@ class basic_json /// @{ private: + /*! + @note Some code in the switch cases has been copied, because otherwise + copilers would complain about implicit fallthrough and there is no + portable attribute to mute such warnings. + */ template static void add_to_vector(std::vector& vec, size_t bytes, const T number) { @@ -6254,24 +7572,31 @@ class basic_json { case 8: { - vec.push_back(static_cast((number >> 070) & 0xff)); - vec.push_back(static_cast((number >> 060) & 0xff)); - vec.push_back(static_cast((number >> 050) & 0xff)); - vec.push_back(static_cast((number >> 040) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((static_cast(number) >> 070) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 060) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 050) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 040) & 0xff)); + vec.push_back(static_cast((number >> 030) & 0xff)); + vec.push_back(static_cast((number >> 020) & 0xff)); + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 4: { vec.push_back(static_cast((number >> 030) & 0xff)); vec.push_back(static_cast((number >> 020) & 0xff)); - // intentional fall-through + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; } case 2: { vec.push_back(static_cast((number >> 010) & 0xff)); - // intentional fall-through + vec.push_back(static_cast(number & 0xff)); + break; } case 1: @@ -6296,7 +7621,7 @@ class basic_json @tparam T the integral return type - @throw std::out_of_range if there are less than sizeof(T)+1 bytes in the + @throw parse_error.110 if there are less than sizeof(T)+1 bytes in the vector @a vec to read In the for loop, the bytes from the vector are copied in reverse order into @@ -6321,13 +7646,11 @@ class basic_json template static T get_from_vector(const std::vector& vec, const size_t current_index) { - if (current_index + sizeof(T) + 1 > vec.size()) - { - throw std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector"); - } + // check if we can read sizeof(T) bytes starting the next index + check_length(vec.size(), sizeof(T), current_index + 1); T result; - uint8_t* ptr = reinterpret_cast(&result); + auto* ptr = reinterpret_cast(&result); for (size_t i = 0; i < sizeof(T); ++i) { *ptr++ = vec[current_index + sizeof(T) - i]; @@ -6368,32 +7691,33 @@ class basic_json if (j.m_value.number_integer >= 0) { // MessagePack does not differentiate between positive - // signed integers and unsigned integers. Therefore, we used - // the code from the value_t::number_unsigned case here. + // signed integers and unsigned integers. Therefore, we + // used the code from the value_t::number_unsigned case + // here. if (j.m_value.number_unsigned < 128) { // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6407,25 +7731,25 @@ class basic_json // negative fixnum add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT8_MIN and j.m_value.number_integer <= INT8_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 v.push_back(0xd0); add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT16_MIN and j.m_value.number_integer <= INT16_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 v.push_back(0xd1); add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT32_MIN and j.m_value.number_integer <= INT32_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 v.push_back(0xd2); add_to_vector(v, 4, j.m_value.number_integer); } - else if (j.m_value.number_integer >= INT64_MIN and j.m_value.number_integer <= INT64_MAX) + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and j.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 v.push_back(0xd3); @@ -6442,25 +7766,25 @@ class basic_json // positive fixnum add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT8_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 v.push_back(0xcc); add_to_vector(v, 1, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT16_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 v.push_back(0xcd); add_to_vector(v, 2, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT32_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 v.push_back(0xce); add_to_vector(v, 4, j.m_value.number_unsigned); } - else if (j.m_value.number_unsigned <= UINT64_MAX) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 v.push_back(0xcf); @@ -6473,7 +7797,7 @@ class basic_json { // float 64 v.push_back(0xcb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6617,19 +7941,19 @@ class basic_json { add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT8_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x18); // one-byte uint8_t add_to_vector(v, 1, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT16_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x19); // two-byte uint16_t add_to_vector(v, 2, j.m_value.number_integer); } - else if (j.m_value.number_integer <= UINT32_MAX) + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) { v.push_back(0x1a); // four-byte uint32_t @@ -6644,26 +7968,26 @@ class basic_json } else { - // The conversions below encode the sign in the first byte, - // and the value is converted to a positive number. + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. const auto positive_number = -1 - j.m_value.number_integer; if (j.m_value.number_integer >= -24) { v.push_back(static_cast(0x20 + positive_number)); } - else if (positive_number <= UINT8_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 8 v.push_back(0x38); add_to_vector(v, 1, positive_number); } - else if (positive_number <= UINT16_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 16 v.push_back(0x39); add_to_vector(v, 2, positive_number); } - else if (positive_number <= UINT32_MAX) + else if (positive_number <= (std::numeric_limits::max)()) { // int 32 v.push_back(0x3a); @@ -6716,7 +8040,7 @@ class basic_json { // Double-Precision Float v.push_back(0xfb); - const uint8_t* helper = reinterpret_cast(&(j.m_value.number_float)); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); for (size_t i = 0; i < 8; ++i) { v.push_back(helper[7 - i]); @@ -6729,7 +8053,7 @@ class basic_json const auto N = j.m_value.string->size(); if (N <= 0x17) { - v.push_back(0x60 + N); // 1 byte for string + size + v.push_back(static_cast(0x60 + N)); // 1 byte for string + size } else if (N <= 0xff) { @@ -6765,7 +8089,7 @@ class basic_json const auto N = j.m_value.array->size(); if (N <= 0x17) { - v.push_back(0x80 + N); // 1 byte for array + size + v.push_back(static_cast(0x80 + N)); // 1 byte for array + size } else if (N <= 0xff) { @@ -6803,7 +8127,7 @@ class basic_json const auto N = j.m_value.object->size(); if (N <= 0x17) { - v.push_back(0xa0 + N); // 1 byte for object + size + v.push_back(static_cast(0xa0 + N)); // 1 byte for object + size } else if (N <= 0xff) { @@ -6850,12 +8174,12 @@ class basic_json To secure the access to the byte vector during CBOR/MessagePack deserialization, bytes are copied from the vector into buffers. This - function checks if the number of bytes to copy (@a len) does not exceed the - size @s size of the vector. Additionally, an @a offset is given from where - to start reading the bytes. + function checks if the number of bytes to copy (@a len) does not exceed + the size @s size of the vector. Additionally, an @a offset is given from + where to start reading the bytes. - This function checks whether reading the bytes is safe; that is, offset is a - valid index in the vector, offset+len + This function checks whether reading the bytes is safe; that is, offset is + a valid index in the vector, offset+len @param[in] size size of the byte vector @param[in] len number of bytes to read @@ -6872,22 +8196,83 @@ class basic_json // simple case: requested length is greater than the vector's length if (len > size or offset > size) { - throw std::out_of_range("len out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // second case: adding offset would result in overflow - if ((size > (std::numeric_limits::max() - offset))) + if ((size > ((std::numeric_limits::max)() - offset))) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } // last case: reading past the end of the vector if (len + offset > size) { - throw std::out_of_range("len+offset out of range"); + JSON_THROW(parse_error::create(110, offset + 1, "cannot read " + std::to_string(len) + " bytes from vector")); } } + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in MessagePack: + + - 0xa0 - 0xbf: fixstr + - 0xd9: str 8 + - 0xda: str 16 + - 0xdb: str 32 + + @param[in] v MessagePack serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void msgpack_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0xa0 and byte <= 0xbf) or (byte >= 0xd9 and byte <= 0xdb)) + { + return; + } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a MessagePack string; last byte: 0x" + ss.str())); + } + + /*! + @brief check if the next byte belongs to a string + + While parsing a map, the keys must be strings. This function checks if the + current byte is one of the start bytes for a string in CBOR: + + - 0x60 - 0x77: fixed length + - 0x78 - 0x7b: variable length + - 0x7f: indefinity length + + @param[in] v CBOR serialization + @param[in] idx byte index in @a v to check for a string + + @throw parse_error.113 if `v[idx]` does not belong to a string + */ + static void cbor_expect_string(const std::vector& v, size_t idx) + { + check_length(v.size(), 1, idx); + + const auto byte = v[idx]; + if ((byte >= 0x60 and byte <= 0x7b) or byte == 0x7f) + { + return; + } + + std::stringstream ss; + ss << std::hex << static_cast(v[idx]); + JSON_THROW(parse_error::create(113, idx + 1, "expected a CBOR string; last byte: 0x" + ss.str())); + } + /*! @brief create a JSON value from a given MessagePack vector @@ -6896,32 +8281,34 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @sa https://github.com/msgpack/msgpack/blob/master/spec.md */ static basic_json from_msgpack_internal(const std::vector& v, size_t& idx) { - // make sure reading 1 byte is safe - check_length(v.size(), 1, idx); - // store and increment index const size_t current_idx = idx++; + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + if (v[current_idx] <= 0xbf) { if (v[current_idx] <= 0x7f) // positive fixint { return v[current_idx]; } - else if (v[current_idx] <= 0x8f) // fixmap + if (v[current_idx] <= 0x8f) // fixmap { basic_json result = value_t::object; const size_t len = v[current_idx] & 0x0f; for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -6972,8 +8359,8 @@ class basic_json case 0xca: // float 32 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -6985,8 +8372,8 @@ class basic_json case 0xcb: // float 64 { // copy bytes in reverse order into the double variable - check_length(v.size(), sizeof(double), 1); double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7101,6 +8488,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7114,6 +8502,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + msgpack_expect_string(v, idx); std::string key = from_msgpack_internal(v, idx); result[key] = from_msgpack_internal(v, idx); } @@ -7122,7 +8511,9 @@ class basic_json default: { - throw std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading MessagePack; last byte: 0x" + ss.str())); } } } @@ -7136,9 +8527,10 @@ class basic_json @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid CBOR - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @sa https://tools.ietf.org/html/rfc7049 */ @@ -7147,7 +8539,10 @@ class basic_json // store and increment index const size_t current_idx = idx++; - switch (v.at(current_idx)) + // make sure reading 1 byte is safe + check_length(v.size(), 1, current_idx); + + switch (v[current_idx]) { // Integer 0x00..0x17 (0..23) case 0x00: @@ -7328,7 +8723,7 @@ class basic_json case 0x7f: // UTF-8 string (indefinite length) { std::string result; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { string_t s = from_cbor_internal(v, idx); result += s; @@ -7424,7 +8819,7 @@ class basic_json case 0x9f: // array (indefinite length) { basic_json result = value_t::array; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { result.push_back(from_cbor_internal(v, idx)); } @@ -7463,6 +8858,7 @@ class basic_json const auto len = static_cast(v[current_idx] - 0xa0); for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7476,6 +8872,7 @@ class basic_json idx += 1; // skip 1 size byte for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7489,6 +8886,7 @@ class basic_json idx += 2; // skip 2 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7502,6 +8900,7 @@ class basic_json idx += 4; // skip 4 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7515,6 +8914,7 @@ class basic_json idx += 8; // skip 8 size bytes for (size_t i = 0; i < len; ++i) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7524,8 +8924,9 @@ class basic_json case 0xbf: // map (indefinite length) { basic_json result = value_t::object; - while (v.at(idx) != 0xff) + while (static_cast(check_length(v.size(), 1, idx)), v[idx] != 0xff) { + cbor_expect_string(v, idx); std::string key = from_cbor_internal(v, idx); result[key] = from_cbor_internal(v, idx); } @@ -7551,7 +8952,6 @@ class basic_json case 0xf9: // Half-Precision Float (two-byte IEEE 754) { - check_length(v.size(), 2, 1); idx += 2; // skip two content bytes // code from RFC 7049, Appendix D, Figure 3: @@ -7561,6 +8961,7 @@ class basic_json // include at least decoding support for them even without such // support. An example of a small decoder for half-precision // floating-point numbers in the C language is shown in Fig. 3. + check_length(v.size(), 2, current_idx + 1); const int half = (v[current_idx + 1] << 8) + v[current_idx + 2]; const int exp = (half >> 10) & 0x1f; const int mant = half & 0x3ff; @@ -7575,16 +8976,18 @@ class basic_json } else { - val = mant == 0 ? INFINITY : NAN; + val = mant == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); } - return half & 0x8000 ? -val : val; + return (half & 0x8000) != 0 ? -val : val; } case 0xfa: // Single-Precision Float (four-byte IEEE 754) { // copy bytes in reverse order into the float variable - check_length(v.size(), sizeof(float), 1); float res; + check_length(v.size(), sizeof(float), current_idx + 1); for (size_t byte = 0; byte < sizeof(float); ++byte) { reinterpret_cast(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte]; @@ -7595,9 +8998,9 @@ class basic_json case 0xfb: // Double-Precision Float (eight-byte IEEE 754) { - check_length(v.size(), sizeof(double), 1); // copy bytes in reverse order into the double variable double res; + check_length(v.size(), sizeof(double), current_idx + 1); for (size_t byte = 0; byte < sizeof(double); ++byte) { reinterpret_cast(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte]; @@ -7608,7 +9011,9 @@ class basic_json default: // anything else (0xFF is handled inside the other types) { - throw std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx]))); + std::stringstream ss; + ss << std::hex << static_cast(v[current_idx]); + JSON_THROW(parse_error::create(112, current_idx + 1, "error reading CBOR; last byte: 0x" + ss.str())); } } } @@ -7621,6 +9026,58 @@ class basic_json serialization format. MessagePack is a binary serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + MessagePack types according to the MessagePack specification: + + JSON value type | value/range | MessagePack type | first byte + --------------- | --------------------------------- | ---------------- | ---------- + null | `null` | nil | 0xc0 + boolean | `true` | true | 0xc3 + boolean | `false` | false | 0xc2 + number_integer | -9223372036854775808..-2147483649 | int64 | 0xd3 + number_integer | -2147483648..-32769 | int32 | 0xd2 + number_integer | -32768..-129 | int16 | 0xd1 + number_integer | -128..-33 | int8 | 0xd0 + number_integer | -32..-1 | negative fixint | 0xe0..0xff + number_integer | 0..127 | positive fixint | 0x00..0x7f + number_integer | 128..255 | uint 8 | 0xcc + number_integer | 256..65535 | uint 16 | 0xcd + number_integer | 65536..4294967295 | uint 32 | 0xce + number_integer | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_unsigned | 0..127 | positive fixint | 0x00..0x7f + number_unsigned | 128..255 | uint 8 | 0xcc + number_unsigned | 256..65535 | uint 16 | 0xcd + number_unsigned | 65536..4294967295 | uint 32 | 0xce + number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xcf + number_float | *any value* | float 64 | 0xcb + string | *length*: 0..31 | fixstr | 0xa0..0xbf + string | *length*: 32..255 | str 8 | 0xd9 + string | *length*: 256..65535 | str 16 | 0xda + string | *length*: 65536..4294967295 | str 32 | 0xdb + array | *size*: 0..15 | fixarray | 0x90..0x9f + array | *size*: 16..65535 | array 16 | 0xdc + array | *size*: 65536..4294967295 | array 32 | 0xdd + object | *size*: 0..15 | fix map | 0x80..0x8f + object | *size*: 16..65535 | map 16 | 0xde + object | *size*: 65536..4294967295 | map 32 | 0xdf + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a MessagePack value. + + @note The following values can **not** be converted to a MessagePack value: + - strings with more than 4294967295 bytes + - arrays with more than 4294967295 elements + - objects with more than 4294967295 elements + + @note The following MessagePack types are not used in the conversion: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - float 32 (0xca) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7630,9 +9087,11 @@ class basic_json vector in MessagePack format.,to_msgpack} @sa http://msgpack.org - @sa @ref from_msgpack(const std::vector&) for the analogous - deserialization + @sa @ref from_msgpack(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + + @since version 2.0.9 */ static std::vector to_msgpack(const basic_json& j) { @@ -7647,12 +9106,54 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the MessagePack serialization format. + The library maps MessagePack types to JSON value types as follows: + + MessagePack type | JSON value type | first byte + ---------------- | --------------- | ---------- + positive fixint | number_unsigned | 0x00..0x7f + fixmap | object | 0x80..0x8f + fixarray | array | 0x90..0x9f + fixstr | string | 0xa0..0xbf + nil | `null` | 0xc0 + false | `false` | 0xc2 + true | `true` | 0xc3 + float 32 | number_float | 0xca + float 64 | number_float | 0xcb + uint 8 | number_unsigned | 0xcc + uint 16 | number_unsigned | 0xcd + uint 32 | number_unsigned | 0xce + uint 64 | number_unsigned | 0xcf + int 8 | number_integer | 0xd0 + int 16 | number_integer | 0xd1 + int 32 | number_integer | 0xd2 + int 64 | number_integer | 0xd3 + str 8 | string | 0xd9 + str 16 | string | 0xda + str 32 | string | 0xdb + array 16 | array | 0xdc + array 32 | array | 0xdd + map 16 | object | 0xde + map 32 | object | 0xdf + negative fixint | number_integer | 0xe0-0xff + + @warning The mapping is **incomplete** in the sense that not all + MessagePack types can be converted to a JSON value. The following + MessagePack types are not supported and will yield parse errors: + - bin 8 - bin 32 (0xc4..0xc6) + - ext 8 - ext 32 (0xc7..0xc9) + - fixext 1 - fixext 16 (0xd4..0xd8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + @param[in] v a byte vector in MessagePack format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from MessagePack were + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from MessagePack were used in the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7661,11 +9162,15 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(const std::vector&) for the related CBOR format + @sa @ref from_cbor(const std::vector&, const size_t) for the + related CBOR format + + @since version 2.0.9, parameter @a start_index since 2.1.1 */ - static basic_json from_msgpack(const std::vector& v) + static basic_json from_msgpack(const std::vector& v, + const size_t start_index = 0) { - size_t i = 0; + size_t i = start_index; return from_msgpack_internal(v, i); } @@ -7677,6 +9182,65 @@ class basic_json serialization format which aims to be more compact than JSON itself, yet more efficient to parse. + The library uses the following mapping from JSON values types to + CBOR types according to the CBOR specification (RFC 7049): + + JSON value type | value/range | CBOR type | first byte + --------------- | ------------------------------------------ | ---------------------------------- | --------------- + null | `null` | Null | 0xf6 + boolean | `true` | True | 0xf5 + boolean | `false` | False | 0xf4 + number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3b + number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3a + number_integer | -32768..-129 | Negative integer (2 bytes follow) | 0x39 + number_integer | -128..-25 | Negative integer (1 byte follow) | 0x38 + number_integer | -24..-1 | Negative integer | 0x20..0x37 + number_integer | 0..23 | Integer | 0x00..0x17 + number_integer | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_integer | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_unsigned | 0..23 | Integer | 0x00..0x17 + number_unsigned | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_unsigned | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1a + number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1b + number_float | *any value* | Double-Precision Float | 0xfb + string | *length*: 0..23 | UTF-8 string | 0x60..0x77 + string | *length*: 23..255 | UTF-8 string (1 byte follow) | 0x78 + string | *length*: 256..65535 | UTF-8 string (2 bytes follow) | 0x79 + string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7a + string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7b + array | *size*: 0..23 | array | 0x80..0x97 + array | *size*: 23..255 | array (1 byte follow) | 0x98 + array | *size*: 256..65535 | array (2 bytes follow) | 0x99 + array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9a + array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9b + object | *size*: 0..23 | map | 0xa0..0xb7 + object | *size*: 23..255 | map (1 byte follow) | 0xb8 + object | *size*: 256..65535 | map (2 bytes follow) | 0xb9 + object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xba + object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xbb + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a CBOR value. + + @note The following CBOR types are not used in the conversion: + - byte strings (0x40..0x5f) + - UTF-8 strings terminated by "break" (0x7f) + - arrays terminated by "break" (0x9f) + - maps terminated by "break" (0xbf) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + - half and single-precision floats (0xf9-0xfa) + - break (0xff) + @param[in] j JSON value to serialize @return MessagePack serialization as byte vector @@ -7686,9 +9250,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector&) for the analogous - deserialization + @sa @ref from_cbor(const std::vector&, const size_t) for the + analogous deserialization @sa @ref to_msgpack(const basic_json& for the related MessagePack format + + @since version 2.0.9 */ static std::vector to_cbor(const basic_json& j) { @@ -7703,12 +9269,74 @@ class basic_json Deserializes a given byte vector @a v to a JSON value using the CBOR (Concise Binary Object Representation) serialization format. + The library maps CBOR types to JSON value types as follows: + + CBOR type | JSON value type | first byte + ---------------------- | --------------- | ---------- + Integer | number_unsigned | 0x00..0x17 + Unsigned integer | number_unsigned | 0x18 + Unsigned integer | number_unsigned | 0x19 + Unsigned integer | number_unsigned | 0x1a + Unsigned integer | number_unsigned | 0x1b + Negative integer | number_integer | 0x20..0x37 + Negative integer | number_integer | 0x38 + Negative integer | number_integer | 0x39 + Negative integer | number_integer | 0x3a + Negative integer | number_integer | 0x3b + Negative integer | number_integer | 0x40..0x57 + UTF-8 string | string | 0x60..0x77 + UTF-8 string | string | 0x78 + UTF-8 string | string | 0x79 + UTF-8 string | string | 0x7a + UTF-8 string | string | 0x7b + UTF-8 string | string | 0x7f + array | array | 0x80..0x97 + array | array | 0x98 + array | array | 0x99 + array | array | 0x9a + array | array | 0x9b + array | array | 0x9f + map | object | 0xa0..0xb7 + map | object | 0xb8 + map | object | 0xb9 + map | object | 0xba + map | object | 0xbb + map | object | 0xbf + False | `false` | 0xf4 + True | `true` | 0xf5 + Nill | `null` | 0xf6 + Half-Precision Float | number_float | 0xf9 + Single-Precision Float | number_float | 0xfa + Double-Precision Float | number_float | 0xfb + + @warning The mapping is **incomplete** in the sense that not all CBOR + types can be converted to a JSON value. The following CBOR types + are not supported and will yield parse errors (parse_error.112): + - byte strings (0x40..0x5f) + - date/time (0xc0..0xc1) + - bignum (0xc2..0xc3) + - decimal fraction (0xc4) + - bigfloat (0xc5) + - tagged items (0xc6..0xd4, 0xd8..0xdb) + - expected conversions (0xd5..0xd7) + - simple values (0xe0..0xf3, 0xf8) + - undefined (0xf7) + + @warning CBOR allows map keys of any type, whereas JSON only allows + strings as keys in object values. Therefore, CBOR maps with keys + other than UTF-8 strings are rejected (parse_error.113). + + @note Any CBOR output created @ref to_cbor can be successfully parsed by + @ref from_cbor. + @param[in] v a byte vector in CBOR format + @param[in] start_index the index to start reading from @a v (0 by default) @return deserialized JSON value - @throw std::invalid_argument if unsupported features from CBOR were used in - the given vector @a v or if the input is not valid MessagePack - @throw std::out_of_range if the given vector ends prematurely + @throw parse_error.110 if the given vector ends prematurely + @throw parse_error.112 if unsupported features from CBOR were + used in the given vector @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found @complexity Linear in the size of the byte vector @a v. @@ -7717,18 +9345,20 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(const std::vector&) for the related - MessagePack format + @sa @ref from_msgpack(const std::vector&, const size_t) for the + related MessagePack format + + @since version 2.0.9, parameter @a start_index since 2.1.1 */ - static basic_json from_cbor(const std::vector& v) + static basic_json from_cbor(const std::vector& v, + const size_t start_index = 0) { - size_t i = 0; + size_t i = start_index; return from_cbor_internal(v, i); } /// @} - private: /////////////////////////// // convenience functions // /////////////////////////// @@ -7743,349 +9373,35 @@ class basic_json @complexity Constant. - @since version 1.0.0 + @liveexample{The following code exemplifies `type_name()` for all JSON + types.,type_name} + + @since version 1.0.0, public since 2.1.0 */ std::string type_name() const { - switch (m_type) { - case value_t::null: - return "null"; - case value_t::object: - return "object"; - case value_t::array: - return "array"; - case value_t::string: - return "string"; - case value_t::boolean: - return "boolean"; - case value_t::discarded: - return "discarded"; - default: - return "number"; - } - } - - /*! - @brief calculates the extra space to escape a JSON string - - @param[in] s the string to escape - @return the number of characters required to escape string @a s - - @complexity Linear in the length of string @a s. - */ - static std::size_t extra_space(const string_t& s) noexcept - { - return std::accumulate(s.begin(), s.end(), size_t{}, - [](size_t res, typename string_t::value_type c) - { - switch (c) + switch (m_type) { - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - return res + 1; - } - + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; default: - { - if (c >= 0x00 and c <= 0x1f) - { - // from c (1 byte) to \uxxxx (6 bytes) - return res + 5; - } - else - { - return res; - } - } - } - }); - } - - /*! - @brief escape a string - - Escape a string by replacing certain special characters by a sequence of - an escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. - - @param[in] s the string to escape - @return the escaped string - - @complexity Linear in the length of string @a s. - */ - static string_t escape_string(const string_t& s) - { - const auto space = extra_space(s); - if (space == 0) - { - return s; - } - - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; - - for (const auto& c : s) - { - switch (c) - { - // quotation mark (0x22) - case '"': - { - result[pos + 1] = '"'; - pos += 2; - break; - } - - // reverse solidus (0x5c) - case '\\': - { - // nothing to change - pos += 2; - break; - } - - // backspace (0x08) - case '\b': - { - result[pos + 1] = 'b'; - pos += 2; - break; - } - - // formfeed (0x0c) - case '\f': - { - result[pos + 1] = 'f'; - pos += 2; - break; - } - - // newline (0x0a) - case '\n': - { - result[pos + 1] = 'n'; - pos += 2; - break; - } - - // carriage return (0x0d) - case '\r': - { - result[pos + 1] = 'r'; - pos += 2; - break; - } - - // horizontal tab (0x09) - case '\t': - { - result[pos + 1] = 't'; - pos += 2; - break; - } - - default: - { - if (c >= 0x00 and c <= 0x1f) - { - // convert a number 0..15 to its hex representation - // (0..f) - static const char hexify[16] = - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - - // print character c as \uxxxx - for (const char m : - { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] - }) - { - result[++pos] = m; - } - - ++pos; - } - else - { - // all other characters are added as-is - result[pos++] = c; - } - break; - } - } - } - - return result; - } - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. Note that - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[out] o stream to write to - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(std::ostream& o, - const bool pretty_print, - const unsigned int indent_step, - const unsigned int current_indent = 0) const - { - // variable to hold indentation for recursive calls - unsigned int new_indent = current_indent; - - switch (m_type) - { - case value_t::object: - { - if (m_value.object->empty()) - { - o << "{}"; - return; - } - - o << "{"; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) - { - if (i != m_value.object->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' ') << "\"" - << escape_string(i->first) << "\":" - << (pretty_print ? " " : ""); - i->second.dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') + "}"; - return; - } - - case value_t::array: - { - if (m_value.array->empty()) - { - o << "[]"; - return; - } - - o << "["; - - // increase indentation - if (pretty_print) - { - new_indent += indent_step; - o << "\n"; - } - - for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) - { - if (i != m_value.array->cbegin()) - { - o << (pretty_print ? ",\n" : ","); - } - o << string_t(new_indent, ' '); - i->dump(o, pretty_print, indent_step, new_indent); - } - - // decrease indentation - if (pretty_print) - { - new_indent -= indent_step; - o << "\n"; - } - - o << string_t(new_indent, ' ') << "]"; - return; - } - - case value_t::string: - { - o << string_t("\"") << escape_string(*m_value.string) << "\""; - return; - } - - case value_t::boolean: - { - o << (m_value.boolean ? "true" : "false"); - return; - } - - case value_t::number_integer: - { - o << m_value.number_integer; - return; - } - - case value_t::number_unsigned: - { - o << m_value.number_unsigned; - return; - } - - case value_t::number_float: - { - if (m_value.number_float == 0) - { - // special case for zero to get "0.0"/"-0.0" - o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); - } - else - { - o << m_value.number_float; - } - return; - } - - case value_t::discarded: - { - o << ""; - return; - } - - case value_t::null: - { - o << "null"; - return; + return "number"; } } } + private: ////////////////////// // member variables // @@ -8115,6 +9431,11 @@ class basic_json class primitive_iterator_t { public: + + difference_type get_value() const noexcept + { + return m_it; + } /// set iterator to a defined beginning void set_begin() noexcept { @@ -8139,16 +9460,89 @@ class basic_json return (m_it == end_value); } - /// return reference to the value to change and compare - operator difference_type& () noexcept + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return lhs.m_it == rhs.m_it; } - /// return value to compare - constexpr operator difference_type () const noexcept + friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept { - return m_it; + return !(lhs == rhs); + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it <= rhs.m_it; + } + + friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it > rhs.m_it; + } + + friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it >= rhs.m_it; + } + + primitive_iterator_t operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) + { + return os << it.m_it; + } + + primitive_iterator_t& operator++() + { + ++m_it; + return *this; + } + + primitive_iterator_t operator++(int) + { + auto result = *this; + m_it++; + return result; + } + + primitive_iterator_t& operator--() + { + --m_it; + return *this; + } + + primitive_iterator_t operator--(int) + { + auto result = *this; + m_it--; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) + { + m_it -= n; + return *this; } private: @@ -8500,7 +9894,7 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: @@ -8509,10 +9903,8 @@ class basic_json { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8545,10 +9937,8 @@ class basic_json { return m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8648,7 +10038,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8690,7 +10080,7 @@ class basic_json // if objects are not the same, the comparison is undefined if (m_object != other.m_object) { - throw std::domain_error("cannot compare iterators of different containers"); + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); } assert(m_object != nullptr); @@ -8699,7 +10089,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot compare order of object iterators"); + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators")); } case basic_json::value_t::array: @@ -8753,7 +10143,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8815,7 +10205,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use offsets with object iterators"); + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); } case basic_json::value_t::array: @@ -8842,7 +10232,7 @@ class basic_json { case basic_json::value_t::object: { - throw std::domain_error("cannot use operator[] for object iterators"); + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators")); } case basic_json::value_t::array: @@ -8852,19 +10242,17 @@ class basic_json case basic_json::value_t::null: { - throw std::out_of_range("cannot get value"); + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } default: { - if (m_it.primitive_iterator == -n) + if (m_it.primitive_iterator.get_value() == -n) { return *m_object; } - else - { - throw std::out_of_range("cannot get value"); - } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); } } } @@ -8881,10 +10269,8 @@ class basic_json { return m_it.object_iterator->first; } - else - { - throw std::domain_error("cannot use key() for non-object iterators"); - } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators")); } /*! @@ -9039,7 +10425,9 @@ class basic_json literal_false, ///< the `false` literal literal_null, ///< the `null` literal value_string, ///< a string -- use get_string() for actual value - value_number, ///< a number -- use get_number() for actual value + value_unsigned, ///< an unsigned integer -- use get_number() for actual value + value_integer, ///< a signed integer -- use get_number() for actual value + value_float, ///< an floating point number -- use get_number() for actual value begin_array, ///< the character for array begin `[` begin_object, ///< the character for object begin `{` end_array, ///< the character for array end `]` @@ -9062,14 +10450,17 @@ class basic_json m_limit = m_content + len; } - /// a lexer from an input stream + /*! + @brief a lexer from an input stream + @throw parse_error.111 if input stream is in a bad state + */ explicit lexer(std::istream& s) : m_stream(&s), m_line_buffer() { // immediately abort if stream is erroneous if (s.fail()) { - throw std::invalid_argument("stream error"); + JSON_THROW(parse_error::create(111, 0, "bad input stream")); } // fill buffer @@ -9103,17 +10494,17 @@ class basic_json @return string representation of the code point; the length of the result string is between 1 and 4 characters. - @throw std::out_of_range if code point is > 0x10ffff; example: `"code - points above 0x10FFFF are invalid"` - @throw std::invalid_argument if the low surrogate is invalid; example: + @throw parse_error.102 if the low surrogate is invalid; example: `""missing or wrong low surrogate""` + @throw parse_error.103 if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` @complexity Constant. @see */ - static string_t to_unicode(const std::size_t codepoint1, - const std::size_t codepoint2 = 0) + string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) const { // calculate the code point from the given code points std::size_t codepoint = codepoint1; @@ -9136,7 +10527,7 @@ class basic_json } else { - throw std::invalid_argument("missing or wrong low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing or wrong low surrogate")); } } @@ -9150,27 +10541,27 @@ class basic_json else if (codepoint <= 0x7ff) { // 2-byte characters: 110xxxxx 10xxxxxx - result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0xC0 | (codepoint >> 6))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0xffff) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0xE0 | (codepoint >> 12))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0x10ffff) { // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0xF0 | (codepoint >> 18))); result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); result.append(1, static_cast(0x80 | (codepoint & 0x3F))); } else { - throw std::out_of_range("code points above 0x10FFFF are invalid"); + JSON_THROW(parse_error::create(103, get_position(), "code points above 0x10FFFF are invalid")); } return result; @@ -9191,7 +10582,9 @@ class basic_json return "null literal"; case token_type::value_string: return "string literal"; - case token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: return "number literal"; case token_type::begin_array: return "'['"; @@ -9233,10 +10626,9 @@ class basic_json Proof (by contradiction): Assume a finite input. To loop forever, the loop must never hit code with a `break` statement. The only code - snippets without a `break` statement are the continue statements for - whitespace and byte-order-marks. To loop forever, the input must be an - infinite sequence of whitespace or byte-order-marks. This contradicts - the assumption of finite input, q.e.d. + snippets without a `break` statement is the continue statement for + whitespace. To loop forever, the input must be an infinite sequence + whitespace. This contradicts the assumption of finite input, q.e.d. */ token_type scan() { @@ -9428,6 +10820,7 @@ basic_json_parser_6: goto basic_json_parser_6; } { + position += static_cast((m_cursor - m_start)); continue; } basic_json_parser_9: @@ -9464,37 +10857,47 @@ basic_json_parser_12: } if (yych <= '0') { - goto basic_json_parser_13; + goto basic_json_parser_43; } if (yych <= '9') { - goto basic_json_parser_15; + goto basic_json_parser_45; } goto basic_json_parser_5; basic_json_parser_13: yyaccept = 1; yych = *(m_marker = ++m_cursor); - if (yych <= 'D') + if (yych <= '9') { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; } } else { if (yych <= 'E') { - goto basic_json_parser_44; + if (yych >= 'E') + { + goto basic_json_parser_51; + } } - if (yych == 'e') + else { - goto basic_json_parser_44; + if (yych == 'e') + { + goto basic_json_parser_51; + } } } basic_json_parser_14: { - last_token_type = token_type::value_number; + last_token_type = token_type::value_unsigned; break; } basic_json_parser_15: @@ -9513,7 +10916,7 @@ basic_json_parser_15: { if (yych == '.') { - goto basic_json_parser_43; + goto basic_json_parser_47; } goto basic_json_parser_14; } @@ -9521,11 +10924,11 @@ basic_json_parser_15: { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } goto basic_json_parser_14; } @@ -9552,7 +10955,7 @@ basic_json_parser_23: yych = *(m_marker = ++m_cursor); if (yych == 'a') { - goto basic_json_parser_45; + goto basic_json_parser_52; } goto basic_json_parser_5; basic_json_parser_24: @@ -9560,7 +10963,7 @@ basic_json_parser_24: yych = *(m_marker = ++m_cursor); if (yych == 'u') { - goto basic_json_parser_46; + goto basic_json_parser_53; } goto basic_json_parser_5; basic_json_parser_25: @@ -9568,7 +10971,7 @@ basic_json_parser_25: yych = *(m_marker = ++m_cursor); if (yych == 'r') { - goto basic_json_parser_47; + goto basic_json_parser_54; } goto basic_json_parser_5; basic_json_parser_26: @@ -9650,13 +11053,27 @@ basic_json_parser_31: } basic_json_parser_32: m_cursor = m_marker; - if (yyaccept == 0) + if (yyaccept <= 1) { - goto basic_json_parser_5; + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } } else { - goto basic_json_parser_14; + if (yyaccept == 2) + { + goto basic_json_parser_44; + } + else + { + goto basic_json_parser_58; + } } basic_json_parser_33: ++m_cursor; @@ -9737,7 +11154,7 @@ basic_json_parser_35: } if (yych <= 'u') { - goto basic_json_parser_48; + goto basic_json_parser_55; } goto basic_json_parser_32; } @@ -9856,64 +11273,159 @@ basic_json_parser_42: } goto basic_json_parser_32; basic_json_parser_43: + yyaccept = 2; + yych = *(m_marker = ++m_cursor); + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; + } + } + else + { + if (yych <= 'E') + { + if (yych >= 'E') + { + goto basic_json_parser_51; + } + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + } + } +basic_json_parser_44: + { + last_token_type = token_type::value_integer; + break; + } +basic_json_parser_45: + yyaccept = 2; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych <= '/') + { + goto basic_json_parser_44; + } + goto basic_json_parser_45; + } + else + { + if (yych <= 'E') + { + if (yych <= 'D') + { + goto basic_json_parser_44; + } + goto basic_json_parser_51; + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + goto basic_json_parser_44; + } + } +basic_json_parser_47: yych = *++m_cursor; if (yych <= '/') { goto basic_json_parser_32; } if (yych <= '9') - { - goto basic_json_parser_49; - } - goto basic_json_parser_32; -basic_json_parser_44: - yych = *++m_cursor; - if (yych <= ',') - { - if (yych == '+') - { - goto basic_json_parser_51; - } - goto basic_json_parser_32; - } - else - { - if (yych <= '-') - { - goto basic_json_parser_51; - } - if (yych <= '/') - { - goto basic_json_parser_32; - } - if (yych <= '9') - { - goto basic_json_parser_52; - } - goto basic_json_parser_32; - } -basic_json_parser_45: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_54; - } - goto basic_json_parser_32; -basic_json_parser_46: - yych = *++m_cursor; - if (yych == 'l') - { - goto basic_json_parser_55; - } - goto basic_json_parser_32; -basic_json_parser_47: - yych = *++m_cursor; - if (yych == 'u') { goto basic_json_parser_56; } goto basic_json_parser_32; basic_json_parser_48: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_50; + } + if (yych <= '9') + { + goto basic_json_parser_48; + } +basic_json_parser_50: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_51: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_59; + } + goto basic_json_parser_32; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_59; + } + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_32; + } +basic_json_parser_52: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_62; + } + goto basic_json_parser_32; +basic_json_parser_53: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_63; + } + goto basic_json_parser_32; +basic_json_parser_54: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_64; + } + goto basic_json_parser_32; +basic_json_parser_55: ++m_cursor; if (m_limit <= m_cursor) { @@ -9928,7 +11440,7 @@ basic_json_parser_48: } if (yych <= '9') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } @@ -9936,7 +11448,7 @@ basic_json_parser_48: { if (yych <= 'F') { - goto basic_json_parser_57; + goto basic_json_parser_65; } if (yych <= '`') { @@ -9944,12 +11456,12 @@ basic_json_parser_48: } if (yych <= 'f') { - goto basic_json_parser_57; + goto basic_json_parser_65; } goto basic_json_parser_32; } -basic_json_parser_49: - yyaccept = 1; +basic_json_parser_56: + yyaccept = 3; m_marker = ++m_cursor; if ((m_limit - m_cursor) < 3) { @@ -9960,27 +11472,30 @@ basic_json_parser_49: { if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_49; + goto basic_json_parser_56; } - goto basic_json_parser_14; } else { if (yych <= 'E') { - goto basic_json_parser_44; + goto basic_json_parser_51; } if (yych == 'e') { - goto basic_json_parser_44; + goto basic_json_parser_51; } - goto basic_json_parser_14; } -basic_json_parser_51: +basic_json_parser_58: + { + last_token_type = token_type::value_float; + break; + } +basic_json_parser_59: yych = *++m_cursor; if (yych <= '/') { @@ -9990,7 +11505,7 @@ basic_json_parser_51: { goto basic_json_parser_32; } -basic_json_parser_52: +basic_json_parser_60: ++m_cursor; if (m_limit <= m_cursor) { @@ -9999,35 +11514,35 @@ basic_json_parser_52: yych = *m_cursor; if (yych <= '/') { - goto basic_json_parser_14; + goto basic_json_parser_58; } if (yych <= '9') { - goto basic_json_parser_52; + goto basic_json_parser_60; } - goto basic_json_parser_14; -basic_json_parser_54: + goto basic_json_parser_58; +basic_json_parser_62: yych = *++m_cursor; if (yych == 's') { - goto basic_json_parser_58; + goto basic_json_parser_66; } goto basic_json_parser_32; -basic_json_parser_55: +basic_json_parser_63: yych = *++m_cursor; if (yych == 'l') { - goto basic_json_parser_59; + goto basic_json_parser_67; } goto basic_json_parser_32; -basic_json_parser_56: +basic_json_parser_64: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_61; + goto basic_json_parser_69; } goto basic_json_parser_32; -basic_json_parser_57: +basic_json_parser_65: ++m_cursor; if (m_limit <= m_cursor) { @@ -10042,7 +11557,7 @@ basic_json_parser_57: } if (yych <= '9') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } @@ -10050,7 +11565,7 @@ basic_json_parser_57: { if (yych <= 'F') { - goto basic_json_parser_63; + goto basic_json_parser_71; } if (yych <= '`') { @@ -10058,30 +11573,30 @@ basic_json_parser_57: } if (yych <= 'f') { - goto basic_json_parser_63; + goto basic_json_parser_71; } goto basic_json_parser_32; } -basic_json_parser_58: +basic_json_parser_66: yych = *++m_cursor; if (yych == 'e') { - goto basic_json_parser_64; + goto basic_json_parser_72; } goto basic_json_parser_32; -basic_json_parser_59: +basic_json_parser_67: ++m_cursor; { last_token_type = token_type::literal_null; break; } -basic_json_parser_61: +basic_json_parser_69: ++m_cursor; { last_token_type = token_type::literal_true; break; } -basic_json_parser_63: +basic_json_parser_71: ++m_cursor; if (m_limit <= m_cursor) { @@ -10096,7 +11611,7 @@ basic_json_parser_63: } if (yych <= '9') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } @@ -10104,7 +11619,7 @@ basic_json_parser_63: { if (yych <= 'F') { - goto basic_json_parser_66; + goto basic_json_parser_74; } if (yych <= '`') { @@ -10112,17 +11627,17 @@ basic_json_parser_63: } if (yych <= 'f') { - goto basic_json_parser_66; + goto basic_json_parser_74; } goto basic_json_parser_32; } -basic_json_parser_64: +basic_json_parser_72: ++m_cursor; { last_token_type = token_type::literal_false; break; } -basic_json_parser_66: +basic_json_parser_74: ++m_cursor; if (m_limit <= m_cursor) { @@ -10161,6 +11676,7 @@ basic_json_parser_66: } + position += static_cast((m_cursor - m_start)); return last_token_type; } @@ -10209,7 +11725,7 @@ basic_json_parser_66: assert(m_marker == nullptr or m_marker <= m_limit); // number of processed characters (p) - const size_t num_processed_chars = static_cast(m_start - m_content); + const auto num_processed_chars = static_cast(m_start - m_content); // offset for m_marker wrt. to m_start const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; // number of unprocessed characters (u) @@ -10219,7 +11735,7 @@ basic_json_parser_66: if (m_stream == nullptr or m_stream->eof()) { // m_start may or may not be pointing into m_line_buffer at - // this point. We trust the standand library to do the right + // this point. We trust the standard library to do the right // thing. See http://stackoverflow.com/q/28142011/266378 m_line_buffer.assign(m_start, m_limit); @@ -10237,6 +11753,13 @@ basic_json_parser_66: m_line_buffer.erase(0, num_processed_chars); // read next line from input stream m_line_buffer_tmp.clear(); + + // check if stream is still good + if (m_stream->fail()) + { + JSON_THROW(parse_error::create(111, 0, "bad input stream")); + } + std::getline(*m_stream, m_line_buffer_tmp, '\n'); // add line with newline symbol to the line buffer @@ -10307,7 +11830,7 @@ basic_json_parser_66: m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This can be rephrased to m_cursor - m_start - 2 > x. With the precondition, we x <= 0, meaning that the loop condition holds - indefinitly if i is always decreased. However, observe that the value + indefinitely if i is always decreased. However, observe that the value of i is strictly increasing with each iteration, as it is incremented by 1 in the iteration expression and never decremented inside the loop body. Hence, the loop condition will eventually be false which @@ -10316,7 +11839,8 @@ basic_json_parser_66: @return string value of current token without opening and closing quotes - @throw std::out_of_range if to_unicode fails + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails */ string_t get_string() const { @@ -10402,7 +11926,7 @@ basic_json_parser_66: // make sure there is a subsequent unicode if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') { - throw std::invalid_argument("missing low surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing low surrogate")); } // get code yyyy from uxxxx\uyyyy @@ -10415,7 +11939,7 @@ basic_json_parser_66: else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF) { // we found a lone low surrogate - throw std::invalid_argument("missing high surrogate"); + JSON_THROW(parse_error::create(102, get_position(), "missing high surrogate")); } else { @@ -10433,59 +11957,155 @@ basic_json_parser_66: return result; } - /*! - @brief parse floating point number - - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). - - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number - */ - long double str_to_float_t(long double* /* type */, char** endptr) const - { - return std::strtold(reinterpret_cast(m_start), endptr); - } /*! - @brief parse floating point number + @brief parse string into a built-in arithmetic type as if the current + locale is POSIX. - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + @note in floating-point case strtod may parse past the token's end - + this is not an error - @param[in,out] endptr recieves a pointer to the first character after - the number - - @return the floating point number + @note any leading blanks are not handled */ - double str_to_float_t(double* /* type */, char** endptr) const + struct strtonum { - return std::strtod(reinterpret_cast(m_start), endptr); - } + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} - /*! - @brief parse floating point number + /*! + @return true iff parsed successfully as number of type T - This function (and its overloads) serves to select the most approprate - standard floating point number parsing function based on the type - supplied via the first parameter. Set this to @a - static_cast(nullptr). + @param[in,out] val shall contain parsed value, or undefined value + if could not parse + */ + template::value>::type> + bool to(T& val) const + { + return parse(val, std::is_integral()); + } - @param[in,out] endptr recieves a pointer to the first character after - the number + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; - @return the floating point number - */ - float str_to_float_t(float* /* type */, char** endptr) const - { - return std::strtof(reinterpret_cast(m_start), endptr); - } + // floating-point conversion + + // overloaded wrappers for strtod/strtof/strtold + // that will be called from parse + static void strtof(float& f, const char* str, char** endptr) + { + f = std::strtof(str, endptr); + } + + static void strtof(double& f, const char* str, char** endptr) + { + f = std::strtod(str, endptr); + } + + static void strtof(long double& f, const char* str, char** endptr) + { + f = std::strtold(str, endptr); + } + + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + // replace decimal separator with locale-specific version, + // when necessary; data will point to either the original + // string, or buf, or tempstr containing the fixed string. + std::string tempstr; + std::array buf; + const size_t len = static_cast(m_end - m_start); + + // lexer will reject empty numbers + assert(len > 0); + + // since dealing with strtod family of functions, we're + // getting the decimal point char from the C locale facilities + // instead of C++'s numpunct facet of the current std::locale + const auto loc = localeconv(); + assert(loc != nullptr); + const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0]; + + const char* data = m_start; + + if (decimal_point_char != '.') + { + const size_t ds_pos = static_cast(std::find(m_start, m_end, '.') - m_start); + + if (ds_pos != len) + { + // copy the data into the local buffer or tempstr, if + // buffer is too small; replace decimal separator, and + // update data to point to the modified bytes + if ((len + 1) < buf.size()) + { + std::copy(m_start, m_end, buf.begin()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } + else + { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } + } + + char* endptr = nullptr; + value = 0; + // this calls appropriate overload depending on T + strtof(value, data, &endptr); + + // parsing was successful iff strtof parsed exactly the number + // of characters determined by the lexer (len) + const bool ok = (endptr == (data + len)); + + if (ok and (value == static_cast(0.0)) and (*data == '-')) + { + // some implementations forget to negate the zero + value = -0.0; + } + + return ok; + } + + // integral conversion + + signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const + { + return std::strtoll(m_start, endptr, 10); + } + + unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const + { + return std::strtoull(m_start, endptr, 10); + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const + { + char* endptr = nullptr; + errno = 0; // these are thread-local + const auto x = parse_integral(&endptr, std::is_signed()); + + // called right overload? + static_assert(std::is_signed() == std::is_signed(), ""); + + value = static_cast(x); + + return (x == static_cast(value)) // x fits into destination T + and (x < 0) == (value < 0) // preserved sign + //and ((x != 0) or is_integral()) // strto[u]ll did nto fail + and (errno == 0) // strto[u]ll did not overflow + and (m_start < m_end) // token was not empty + and (endptr == m_end); // parsed entire token exactly + } + }; /*! @brief return number value for number tokens @@ -10494,125 +12114,88 @@ basic_json_parser_66: number type (either integer, unsigned integer or floating point), which is passed back to the caller via the result parameter. - This function parses the integer component up to the radix point or - exponent while collecting information about the 'floating point - representation', which it stores in the result parameter. If there is - no radix point or exponent, and the number can fit into a @ref - number_integer_t or @ref number_unsigned_t then it sets the result - parameter accordingly. + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t - If the number is a floating point number the number is then parsed - using @a std:strtod (or @a std:strtof or @a std::strtold). + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null - @param[out] result @ref basic_json object to receive the number, or - NAN if the conversion read past the current token. The latter case - needs to be treated by the caller function. + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number + + @param[out] result @ref basic_json object to receive the number. + @param[in] token the type of the number token */ - void get_number(basic_json& result) const + bool get_number(basic_json& result, const token_type token) const { assert(m_start != nullptr); + assert(m_start < m_cursor); + assert((token == token_type::value_unsigned) or + (token == token_type::value_integer) or + (token == token_type::value_float)); - const lexer::lexer_char_t* curptr = m_start; + strtonum num_converter(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); - // accumulate the integer conversion result (unsigned for now) - number_unsigned_t value = 0; - - // maximum absolute value of the relevant integer type - number_unsigned_t max; - - // temporarily store the type to avoid unecessary bitfield access - value_t type; - - // look for sign - if (*curptr == '-') + switch (token) { - type = value_t::number_integer; - max = static_cast((std::numeric_limits::max)()) + 1; - curptr++; - } - else - { - type = value_t::number_unsigned; - max = static_cast((std::numeric_limits::max)()); - } - - // count the significant figures - for (; curptr < m_cursor; curptr++) - { - // quickly skip tests if a digit - if (*curptr < '0' || *curptr > '9') + case lexer::token_type::value_unsigned: { - if (*curptr == '.') + number_unsigned_t val; + if (num_converter.to(val)) { - // don't count '.' but change to float - type = value_t::number_float; - continue; + // parsing successful + result.m_type = value_t::number_unsigned; + result.m_value = val; + return true; } - // assume exponent (if not then will fail parse): change to - // float, stop counting and record exponent details - type = value_t::number_float; break; } - // skip if definitely not an integer - if (type != value_t::number_float) + case lexer::token_type::value_integer: { - auto digit = static_cast(*curptr - '0'); - - // overflow if value * 10 + digit > max, move terms around - // to avoid overflow in intermediate values - if (value > (max - digit) / 10) + number_integer_t val; + if (num_converter.to(val)) { - // overflow - type = value_t::number_float; - } - else - { - // no overflow - value = value * 10 + digit; + // parsing successful + result.m_type = value_t::number_integer; + result.m_value = val; + return true; } + break; } - } - // save the value (if not a float) - if (type == value_t::number_unsigned) - { - result.m_value.number_unsigned = value; - } - else if (type == value_t::number_integer) - { - // invariant: if we parsed a '-', the absolute value is between - // 0 (we allow -0) and max == -INT64_MIN - assert(value >= 0); - assert(value <= max); - - if (value == max) + default: { - // we cannot simply negate value (== max == -INT64_MIN), - // see https://github.com/nlohmann/json/issues/389 - result.m_value.number_integer = static_cast(INT64_MIN); - } - else - { - // all other values can be negated safely - result.m_value.number_integer = -static_cast(value); + break; } } - else - { - // parse with strtod - result.m_value.number_float = str_to_float_t(static_cast(nullptr), NULL); - // replace infinity and NAN by null + // parse float (either explicitly or because a previous conversion + // failed) + number_float_t val; + if (num_converter.to(val)) + { + // parsing successful + result.m_type = value_t::number_float; + result.m_value = val; + + // throw in case of infinity or NAN if (not std::isfinite(result.m_value.number_float)) { - type = value_t::null; - result.m_value = basic_json::json_value(); + JSON_THROW(out_of_range::create(406, "number overflow parsing '" + get_token_string() + "'")); } + + return true; } - // save the type - result.m_type = type; + // couldn't parse number in any format + return false; + } + + constexpr size_t get_position() const + { + return position; } private: @@ -10634,6 +12217,8 @@ basic_json_parser_66: const lexer_char_t* m_limit = nullptr; /// the last token type token_type last_token_type = token_type::end_of_input; + /// current position in the input (read bytes) + size_t position = 0; }; /*! @@ -10650,7 +12235,10 @@ basic_json_parser_66: m_lexer(reinterpret_cast(buff), std::strlen(buff)) {} - /// a parser reading from an input stream + /*! + @brief a parser reading from an input stream + @throw parse_error.111 if input stream is in a bad state + */ parser(std::istream& is, const parser_callback_t cb = nullptr) : callback(cb), m_lexer(is) {} @@ -10666,7 +12254,12 @@ basic_json_parser_66: static_cast(std::distance(first, last))) {} - /// public parser interface + /*! + @brief public parser interface + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse() { // read first token @@ -10683,7 +12276,12 @@ basic_json_parser_66: } private: - /// the actual parser + /*! + @brief the actual parser + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ basic_json parse_internal(bool keep) { auto result = basic_json(value_t::discarded); @@ -10856,9 +12454,11 @@ basic_json_parser_66: break; } - case lexer::token_type::value_number: + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: { - m_lexer.get_number(result); + m_lexer.get_number(result, last_token); get_token(); break; } @@ -10884,6 +12484,9 @@ basic_json_parser_66: return last_token; } + /*! + @throw parse_error.101 if expected token did not occur + */ void expect(typename lexer::token_type t) const { if (t != last_token) @@ -10893,10 +12496,13 @@ basic_json_parser_66: "'") : lexer::token_type_name(last_token)); error_msg += "; expected " + lexer::token_type_name(t); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } + /*! + @throw parse_error.101 if unexpected token occurred + */ void unexpect(typename lexer::token_type t) const { if (t == last_token) @@ -10905,7 +12511,7 @@ basic_json_parser_66: error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + "'") : lexer::token_type_name(last_token)); - throw std::invalid_argument(error_msg); + JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); } } @@ -10948,12 +12554,12 @@ basic_json_parser_66: empty string is assumed which references the whole JSON value - @throw std::domain_error if reference token is nonempty and does not - begin with a slash (`/`); example: `"JSON pointer must be empty or - begin with /"` - @throw std::domain_error if a tilde (`~`) is not followed by `0` - (representing `~`) or `1` (representing `/`); example: `"escape error: - ~ must be followed with 0 or 1"` + @throw parse_error.107 if the given JSON pointer @a s is nonempty and + does not begin with a slash (`/`); see example below + + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s + is not followed by `0` (representing `~`) or `1` (representing `/`); + see example below @liveexample{The example shows the construction several valid JSON pointers as well as the exceptional behavior.,json_pointer} @@ -10996,12 +12602,15 @@ basic_json_parser_66: } private: - /// remove and return last reference pointer + /*! + @brief remove and return last reference pointer + @throw out_of_range.405 if JSON pointer has no parent + */ std::string pop_back() { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } auto last = reference_tokens.back(); @@ -11019,7 +12628,7 @@ basic_json_parser_66: { if (is_root()) { - throw std::domain_error("JSON pointer has no parent"); + JSON_THROW(out_of_range::create(405, "JSON pointer has no parent")); } json_pointer result = *this; @@ -11031,6 +12640,9 @@ basic_json_parser_66: @brief create and return a reference to the pointed to value @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened */ reference get_and_create(reference j) const { @@ -11067,7 +12679,14 @@ basic_json_parser_66: case value_t::array: { // create an entry in the array - result = &result->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + result = &result->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } @@ -11080,7 +12699,7 @@ basic_json_parser_66: */ default: { - throw std::domain_error("invalid value to unflatten"); + JSON_THROW(type_error::create(313, "invalid value to unflatten")); } } } @@ -11103,9 +12722,9 @@ basic_json_parser_66: @complexity Linear in the length of the JSON pointer. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved */ reference get_unchecked(pointer ptr) const { @@ -11119,7 +12738,7 @@ basic_json_parser_66: reference_token.end(), [](const char x) { - return std::isdigit(x); + return (x >= '0' and x <= '9'); }); // change value to array for numbers or "-" or to object @@ -11148,25 +12767,32 @@ basic_json_parser_66: // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } if (reference_token == "-") { - // explicityly treat "-" as index beyond the end + // explicitly treat "-" as index beyond the end ptr = &ptr->operator[](ptr->m_value.array->size()); } else { // convert array index to number; unchecked access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11174,6 +12800,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ reference get_checked(pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11192,25 +12824,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11225,6 +12864,11 @@ basic_json_parser_66: @return const reference to the JSON value pointed to by the JSON pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved */ const_reference get_unchecked(const_pointer ptr) const { @@ -11244,25 +12888,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" cannot be used for const access - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // use unchecked array access - ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11270,6 +12921,12 @@ basic_json_parser_66: return *ptr; } + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ const_reference get_checked(const_pointer ptr) const { for (const auto& reference_token : reference_tokens) @@ -11288,25 +12945,32 @@ basic_json_parser_66: if (reference_token == "-") { // "-" always fails the range check - throw std::out_of_range("array index '-' (" + - std::to_string(ptr->m_value.array->size()) + - ") is out of range"); + JSON_THROW(out_of_range::create(402, "array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); } // error condition (cf. RFC 6901, Sect. 4) if (reference_token.size() > 1 and reference_token[0] == '0') { - throw std::domain_error("array index must not begin with '0'"); + JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'")); } // note: at performs range check - ptr = &ptr->at(static_cast(std::stoi(reference_token))); + JSON_TRY + { + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + } + JSON_CATCH (std::invalid_argument&) + { + JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } break; } default: { - throw std::out_of_range("unresolved reference token '" + reference_token + "'"); + JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } } } @@ -11314,7 +12978,15 @@ basic_json_parser_66: return *ptr; } - /// split the string input to reference tokens + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ static std::vector split(const std::string& reference_string) { std::vector result; @@ -11328,7 +13000,7 @@ basic_json_parser_66: // check if nonempty reference string begins with slash if (reference_string[0] != '/') { - throw std::domain_error("JSON pointer must be empty or begin with '/'"); + JSON_THROW(parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'")); } // extract the reference tokens: @@ -11336,7 +13008,7 @@ basic_json_parser_66: // - start: position after the previous slash for ( // search for the first slash after the first character - size_t slash = reference_string.find_first_of("/", 1), + size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; // we can stop if start == string::npos+1 = 0 @@ -11345,16 +13017,16 @@ basic_json_parser_66: // (will eventually be 0 if slash == std::string::npos) start = slash + 1, // find next slash - slash = reference_string.find_first_of("/", start)) + slash = reference_string.find_first_of('/', start)) { // use the text between the beginning of the reference token // (start) and the last slash (slash). auto reference_token = reference_string.substr(start, slash - start); // check reference tokens are properly escaped - for (size_t pos = reference_token.find_first_of("~"); + for (size_t pos = reference_token.find_first_of('~'); pos != std::string::npos; - pos = reference_token.find_first_of("~", pos + 1)) + pos = reference_token.find_first_of('~', pos + 1)) { assert(reference_token[pos] == '~'); @@ -11363,7 +13035,7 @@ basic_json_parser_66: (reference_token[pos + 1] != '0' and reference_token[pos + 1] != '1')) { - throw std::domain_error("escape error: '~' must be followed with '0' or '1'"); + JSON_THROW(parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); } } @@ -11375,7 +13047,6 @@ basic_json_parser_66: return result; } - private: /*! @brief replace all occurrences of a substring by another string @@ -11384,7 +13055,8 @@ basic_json_parser_66: @param[in] f the substring to replace with @a t @param[in] t the string to replace @a f - @pre The search string @a f must not be empty. + @pre The search string @a f must not be empty. **This precondition is + enforced with an assertion.** @since version 2.0.0 */ @@ -11484,12 +13156,17 @@ basic_json_parser_66: @param[in] value flattened JSON @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened */ static basic_json unflatten(const basic_json& value) { if (not value.is_object()) { - throw std::domain_error("only objects can be unflattened"); + JSON_THROW(type_error::create(314, "only objects can be unflattened")); } basic_json result; @@ -11499,7 +13176,7 @@ basic_json_parser_66: { if (not element.second.is_primitive()) { - throw std::domain_error("values in object must be primitive"); + JSON_THROW(type_error::create(315, "values in object must be primitive")); } // assign value to reference pointed to by JSON pointer; Note @@ -11513,7 +13190,18 @@ basic_json_parser_66: return result; } - private: + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + /// the reference tokens std::vector reference_tokens {}; }; @@ -11550,9 +13238,9 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer} @@ -11577,9 +13265,10 @@ basic_json_parser_66: @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} @@ -11600,15 +13289,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. + + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. + + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @since version 2.0.0 @liveexample{The behavior is shown in the example.,at_json_pointer} - - @since version 2.0.0 */ reference at(const json_pointer& ptr) { @@ -11625,15 +13329,30 @@ basic_json_parser_66: @return reference to the element pointed to by @a ptr + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. + + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. + + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + @complexity Constant. - @throw std::out_of_range if the JSON pointer can not be resolved - @throw std::domain_error if an array index begins with '0' - @throw std::invalid_argument if an array index was not a number + @since version 2.0.0 @liveexample{The behavior is shown in the example.,at_json_pointer_const} - - @since version 2.0.0 */ const_reference at(const json_pointer& ptr) const { @@ -11648,7 +13367,7 @@ basic_json_parser_66: primitive. The original JSON value can be restored using the @ref unflatten() function. - @return an object that maps JSON pointers to primitve values + @return an object that maps JSON pointers to primitive values @note Empty objects and arrays are flattened to `null` and will not be reconstructed correctly by the @ref unflatten() function. @@ -11689,6 +13408,9 @@ basic_json_parser_66: @complexity Linear in the size the JSON value. + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitve + @liveexample{The following code shows how a flattened JSON object is unflattened into the original nested JSON object.,unflatten} @@ -11715,7 +13437,7 @@ basic_json_parser_66: [JSON Patch](http://jsonpatch.com) defines a JSON document structure for expressing a sequence of operations to apply to a JSON) document. With - this funcion, a JSON Patch is applied to the current JSON value by + this function, a JSON Patch is applied to the current JSON value by executing all operations from the patch. @param[in] json_patch JSON patch document @@ -11726,12 +13448,23 @@ basic_json_parser_66: any case, the original value is not changed: the patch is applied to a copy of the value. - @throw std::out_of_range if a JSON pointer inside the patch could not - be resolved successfully in the current JSON value; example: `"key baz - not found"` - @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + @throw parse_error.104 if the JSON patch does not consist of an array of + objects + + @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory attributes are missing); example: `"operation add must have member path"` + @throw out_of_range.401 if an array index is out of range. + + @throw out_of_range.403 if a JSON pointer inside the patch could not be + resolved successfully in the current JSON value; example: `"key baz not + found"` + + @throw out_of_range.405 if JSON pointer has no parent ("add", "remove", + "move") + + @throw other_error.501 if "test" operation was unsuccessful + @complexity Linear in the size of the JSON value and the length of the JSON patch. As usually only a fraction of the JSON value is affected by the patch, the complexity can usually be neglected. @@ -11754,7 +13487,7 @@ basic_json_parser_66: // the valid JSON Patch operations enum class patch_operations {add, remove, replace, move, copy, test, invalid}; - const auto get_op = [](const std::string op) + const auto get_op = [](const std::string & op) { if (op == "add") { @@ -11828,7 +13561,7 @@ basic_json_parser_66: if (static_cast(idx) > parent.size()) { // avoid undefined behavior - throw std::out_of_range("array index " + std::to_string(idx) + " is out of range"); + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } else { @@ -11866,7 +13599,7 @@ basic_json_parser_66: } else { - throw std::out_of_range("key '" + last_path + "' not found"); + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found")); } } else if (parent.is_array()) @@ -11876,14 +13609,13 @@ basic_json_parser_66: } }; - // type check + // type check: top level value must be an array if (not json_patch.is_array()) { - // a JSON patch must be an array of objects - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } - // iterate and apply th eoperations + // iterate and apply the operations for (const auto& val : json_patch) { // wrapper to get a value for an operation @@ -11900,23 +13632,23 @@ basic_json_parser_66: // check if desired value is present if (it == val.m_value.object->end()) { - throw std::invalid_argument(error_msg + " must have member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'")); } // check if result is of type string if (string_type and not it->second.is_string()) { - throw std::invalid_argument(error_msg + " must have string member '" + member + "'"); + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'")); } // no error: return value return it->second; }; - // type check + // type check: every element of the array must be an object if (not val.is_object()) { - throw std::invalid_argument("JSON patch must be an array of objects"); + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); } // collect mandatory members @@ -11975,13 +13707,13 @@ basic_json_parser_66: case patch_operations::test: { bool success = false; - try + JSON_TRY { // check if "value" matches the one at "path" // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - catch (std::out_of_range&) + JSON_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -11989,7 +13721,7 @@ basic_json_parser_66: // throw an exception if test fails if (not success) { - throw std::domain_error("unsuccessful: " + val.dump()); + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump())); } break; @@ -11999,7 +13731,7 @@ basic_json_parser_66: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - throw std::invalid_argument("operation value '" + op + "' is invalid"); + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid")); } } } @@ -12022,8 +13754,8 @@ basic_json_parser_66: @note Currently, only `remove`, `add`, and `replace` operations are generated. - @param[in] source JSON value to copare from - @param[in] target JSON value to copare against + @param[in] source JSON value to compare from + @param[in] target JSON value to compare against @param[in] path helper value to create JSON pointers @return a JSON patch to convert the @a source to @a target @@ -12174,7 +13906,6 @@ basic_json_parser_66: /// @} }; - ///////////// // presets // ///////////// @@ -12188,7 +13919,7 @@ uses the standard template types. @since version 1.0.0 */ using json = basic_json<>; -} +} // namespace nlohmann /////////////////////// @@ -12229,7 +13960,23 @@ struct hash return h(j.dump()); } }; -} + +/// specialization for std::less +template <> +struct less<::nlohmann::detail::value_t> +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +} // namespace std /*! @brief user-defined string literal for JSON values @@ -12271,5 +14018,14 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop #endif +#if defined(__clang__) + #pragma GCC diagnostic pop +#endif + +// clean up +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_DEPRECATED #endif diff --git a/zto/ext/x64-salsa2012-asm/README.md b/zto/ext/x64-salsa2012-asm/README.md new file mode 100644 index 0000000..a69a1a6 --- /dev/null +++ b/zto/ext/x64-salsa2012-asm/README.md @@ -0,0 +1,6 @@ +Blazingly fast X64 ASM implementation of Salsa20/12 +====== + +This is ripped from the [cnacl](https://github.com/cjdelisle/cnacl) source. The actual code is by Danial J. Bernstein and is in the public domain. + +This is included on Linux and Mac 64-bit builds and is significantly faster than the SSE intrinsics or C versions. It's used for packet encode/decode only since its use differs a bit from the regular Salsa20 C++ class. Specifically it lacks the ability to be called on multiple blocks, preferring instead to take a key and a single stream to encrypt and that's it. diff --git a/zto/ext/x64-salsa2012-asm/salsa2012.h b/zto/ext/x64-salsa2012-asm/salsa2012.h new file mode 100644 index 0000000..73e375e --- /dev/null +++ b/zto/ext/x64-salsa2012-asm/salsa2012.h @@ -0,0 +1,16 @@ +#ifndef ZT_X64_SALSA2012_ASM +#define ZT_X64_SALSA2012_ASM + +#ifdef __cplusplus +extern "C" { +#endif + +// Generates Salsa20/12 key stream +// output, outlen, nonce, key (256-bit / 32-byte) +extern int zt_salsa2012_amd64_xmm6(unsigned char *, unsigned long long, const unsigned char *, const unsigned char *); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/zto/ext/x64-salsa2012-asm/salsa2012.s b/zto/ext/x64-salsa2012-asm/salsa2012.s new file mode 100644 index 0000000..699c89a --- /dev/null +++ b/zto/ext/x64-salsa2012-asm/salsa2012.s @@ -0,0 +1,4488 @@ +# qhasm: enter zt_salsa2012_amd64_xmm6 +.text +.p2align 5 +.globl _zt_salsa2012_amd64_xmm6 +.globl zt_salsa2012_amd64_xmm6 +_zt_salsa2012_amd64_xmm6: +zt_salsa2012_amd64_xmm6: +mov %rsp,%r11 +and $31,%r11 +add $480,%r11 +sub %r11,%rsp + +# qhasm: r11_stack = r11_caller +# asm 1: movq r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: bytes = arg2 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rsi,%r9 + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: iv = arg3 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rdx,%rdx + +# qhasm: k = arg4 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %rcx,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done + +# qhasm: a = 0 +# asm 1: mov $0,>a=int64#7 +# asm 2: mov $0,>a=%rax +mov $0,%rax + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = a; --i } +rep stosb + +# qhasm: out -= bytes +# asm 1: sub r11_stack=stack64#1 +# asm 2: movq r11_stack=352(%rsp) +movq %r11,352(%rsp) + +# qhasm: r12_stack = r12_caller +# asm 1: movq r12_stack=stack64#2 +# asm 2: movq r12_stack=360(%rsp) +movq %r12,360(%rsp) + +# qhasm: r13_stack = r13_caller +# asm 1: movq r13_stack=stack64#3 +# asm 2: movq r13_stack=368(%rsp) +movq %r13,368(%rsp) + +# qhasm: r14_stack = r14_caller +# asm 1: movq r14_stack=stack64#4 +# asm 2: movq r14_stack=376(%rsp) +movq %r14,376(%rsp) + +# qhasm: r15_stack = r15_caller +# asm 1: movq r15_stack=stack64#5 +# asm 2: movq r15_stack=384(%rsp) +movq %r15,384(%rsp) + +# qhasm: rbx_stack = rbx_caller +# asm 1: movq rbx_stack=stack64#6 +# asm 2: movq rbx_stack=392(%rsp) +movq %rbx,392(%rsp) + +# qhasm: rbp_stack = rbp_caller +# asm 1: movq rbp_stack=stack64#7 +# asm 2: movq rbp_stack=400(%rsp) +movq %rbp,400(%rsp) + +# qhasm: out = arg1 +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdi,%rdi + +# qhasm: m = arg2 +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rsi,%rsi + +# qhasm: bytes = arg3 +# asm 1: mov bytes=int64#6 +# asm 2: mov bytes=%r9 +mov %rdx,%r9 + +# qhasm: iv = arg4 +# asm 1: mov iv=int64#3 +# asm 2: mov iv=%rdx +mov %rcx,%rdx + +# qhasm: k = arg5 +# asm 1: mov k=int64#8 +# asm 2: mov k=%r10 +mov %r8,%r10 + +# qhasm: unsigned>? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: start: +._start: + +# qhasm: in12 = *(uint32 *) (k + 20) +# asm 1: movl 20(in12=int64#4d +# asm 2: movl 20(in12=%ecx +movl 20(%r10),%ecx + +# qhasm: in1 = *(uint32 *) (k + 0) +# asm 1: movl 0(in1=int64#5d +# asm 2: movl 0(in1=%r8d +movl 0(%r10),%r8d + +# qhasm: in6 = *(uint32 *) (iv + 0) +# asm 1: movl 0(in6=int64#7d +# asm 2: movl 0(in6=%eax +movl 0(%rdx),%eax + +# qhasm: in11 = *(uint32 *) (k + 16) +# asm 1: movl 16(in11=int64#9d +# asm 2: movl 16(in11=%r11d +movl 16(%r10),%r11d + +# qhasm: ((uint32 *)&x1)[0] = in12 +# asm 1: movl x1=stack128#1 +# asm 2: movl x1=0(%rsp) +movl %ecx,0(%rsp) + +# qhasm: ((uint32 *)&x1)[1] = in1 +# asm 1: movl in8=int64#4 +# asm 2: mov $0,>in8=%rcx +mov $0,%rcx + +# qhasm: in13 = *(uint32 *) (k + 24) +# asm 1: movl 24(in13=int64#5d +# asm 2: movl 24(in13=%r8d +movl 24(%r10),%r8d + +# qhasm: in2 = *(uint32 *) (k + 4) +# asm 1: movl 4(in2=int64#7d +# asm 2: movl 4(in2=%eax +movl 4(%r10),%eax + +# qhasm: in7 = *(uint32 *) (iv + 4) +# asm 1: movl 4(in7=int64#3d +# asm 2: movl 4(in7=%edx +movl 4(%rdx),%edx + +# qhasm: ((uint32 *)&x2)[0] = in8 +# asm 1: movl x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x2)[1] = in13 +# asm 1: movl in4=int64#3d +# asm 2: movl 12(in4=%edx +movl 12(%r10),%edx + +# qhasm: in9 = 0 +# asm 1: mov $0,>in9=int64#4 +# asm 2: mov $0,>in9=%rcx +mov $0,%rcx + +# qhasm: in14 = *(uint32 *) (k + 28) +# asm 1: movl 28(in14=int64#5d +# asm 2: movl 28(in14=%r8d +movl 28(%r10),%r8d + +# qhasm: in3 = *(uint32 *) (k + 8) +# asm 1: movl 8(in3=int64#7d +# asm 2: movl 8(in3=%eax +movl 8(%r10),%eax + +# qhasm: ((uint32 *)&x3)[0] = in4 +# asm 1: movl x3=stack128#3 +# asm 2: movl x3=32(%rsp) +movl %edx,32(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl in0=int64#3 +# asm 2: mov $1634760805,>in0=%rdx +mov $1634760805,%rdx + +# qhasm: in5 = 857760878 +# asm 1: mov $857760878,>in5=int64#4 +# asm 2: mov $857760878,>in5=%rcx +mov $857760878,%rcx + +# qhasm: in10 = 2036477234 +# asm 1: mov $2036477234,>in10=int64#5 +# asm 2: mov $2036477234,>in10=%r8 +mov $2036477234,%r8 + +# qhasm: in15 = 1797285236 +# asm 1: mov $1797285236,>in15=int64#7 +# asm 2: mov $1797285236,>in15=%rax +mov $1797285236,%rax + +# qhasm: ((uint32 *)&x0)[0] = in0 +# asm 1: movl x0=stack128#4 +# asm 2: movl x0=48(%rsp) +movl %edx,48(%rsp) + +# qhasm: ((uint32 *)&x0)[1] = in5 +# asm 1: movl z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: z5 = z0[1,1,1,1] +# asm 1: pshufd $0x55,z5=int6464#2 +# asm 2: pshufd $0x55,z5=%xmm1 +pshufd $0x55,%xmm0,%xmm1 + +# qhasm: z10 = z0[2,2,2,2] +# asm 1: pshufd $0xaa,z10=int6464#3 +# asm 2: pshufd $0xaa,z10=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z15 = z0[3,3,3,3] +# asm 1: pshufd $0xff,z15=int6464#4 +# asm 2: pshufd $0xff,z15=%xmm3 +pshufd $0xff,%xmm0,%xmm3 + +# qhasm: z0 = z0[0,0,0,0] +# asm 1: pshufd $0x00,z0=int6464#1 +# asm 2: pshufd $0x00,z0=%xmm0 +pshufd $0x00,%xmm0,%xmm0 + +# qhasm: orig5 = z5 +# asm 1: movdqa orig5=stack128#5 +# asm 2: movdqa orig5=64(%rsp) +movdqa %xmm1,64(%rsp) + +# qhasm: orig10 = z10 +# asm 1: movdqa orig10=stack128#6 +# asm 2: movdqa orig10=80(%rsp) +movdqa %xmm2,80(%rsp) + +# qhasm: orig15 = z15 +# asm 1: movdqa orig15=stack128#7 +# asm 2: movdqa orig15=96(%rsp) +movdqa %xmm3,96(%rsp) + +# qhasm: orig0 = z0 +# asm 1: movdqa orig0=stack128#8 +# asm 2: movdqa orig0=112(%rsp) +movdqa %xmm0,112(%rsp) + +# qhasm: z1 = x1 +# asm 1: movdqa z1=int6464#1 +# asm 2: movdqa z1=%xmm0 +movdqa 0(%rsp),%xmm0 + +# qhasm: z6 = z1[2,2,2,2] +# asm 1: pshufd $0xaa,z6=int6464#2 +# asm 2: pshufd $0xaa,z6=%xmm1 +pshufd $0xaa,%xmm0,%xmm1 + +# qhasm: z11 = z1[3,3,3,3] +# asm 1: pshufd $0xff,z11=int6464#3 +# asm 2: pshufd $0xff,z11=%xmm2 +pshufd $0xff,%xmm0,%xmm2 + +# qhasm: z12 = z1[0,0,0,0] +# asm 1: pshufd $0x00,z12=int6464#4 +# asm 2: pshufd $0x00,z12=%xmm3 +pshufd $0x00,%xmm0,%xmm3 + +# qhasm: z1 = z1[1,1,1,1] +# asm 1: pshufd $0x55,z1=int6464#1 +# asm 2: pshufd $0x55,z1=%xmm0 +pshufd $0x55,%xmm0,%xmm0 + +# qhasm: orig6 = z6 +# asm 1: movdqa orig6=stack128#9 +# asm 2: movdqa orig6=128(%rsp) +movdqa %xmm1,128(%rsp) + +# qhasm: orig11 = z11 +# asm 1: movdqa orig11=stack128#10 +# asm 2: movdqa orig11=144(%rsp) +movdqa %xmm2,144(%rsp) + +# qhasm: orig12 = z12 +# asm 1: movdqa orig12=stack128#11 +# asm 2: movdqa orig12=160(%rsp) +movdqa %xmm3,160(%rsp) + +# qhasm: orig1 = z1 +# asm 1: movdqa orig1=stack128#12 +# asm 2: movdqa orig1=176(%rsp) +movdqa %xmm0,176(%rsp) + +# qhasm: z2 = x2 +# asm 1: movdqa z2=int6464#1 +# asm 2: movdqa z2=%xmm0 +movdqa 16(%rsp),%xmm0 + +# qhasm: z7 = z2[3,3,3,3] +# asm 1: pshufd $0xff,z7=int6464#2 +# asm 2: pshufd $0xff,z7=%xmm1 +pshufd $0xff,%xmm0,%xmm1 + +# qhasm: z13 = z2[1,1,1,1] +# asm 1: pshufd $0x55,z13=int6464#3 +# asm 2: pshufd $0x55,z13=%xmm2 +pshufd $0x55,%xmm0,%xmm2 + +# qhasm: z2 = z2[2,2,2,2] +# asm 1: pshufd $0xaa,z2=int6464#1 +# asm 2: pshufd $0xaa,z2=%xmm0 +pshufd $0xaa,%xmm0,%xmm0 + +# qhasm: orig7 = z7 +# asm 1: movdqa orig7=stack128#13 +# asm 2: movdqa orig7=192(%rsp) +movdqa %xmm1,192(%rsp) + +# qhasm: orig13 = z13 +# asm 1: movdqa orig13=stack128#14 +# asm 2: movdqa orig13=208(%rsp) +movdqa %xmm2,208(%rsp) + +# qhasm: orig2 = z2 +# asm 1: movdqa orig2=stack128#15 +# asm 2: movdqa orig2=224(%rsp) +movdqa %xmm0,224(%rsp) + +# qhasm: z3 = x3 +# asm 1: movdqa z3=int6464#1 +# asm 2: movdqa z3=%xmm0 +movdqa 32(%rsp),%xmm0 + +# qhasm: z4 = z3[0,0,0,0] +# asm 1: pshufd $0x00,z4=int6464#2 +# asm 2: pshufd $0x00,z4=%xmm1 +pshufd $0x00,%xmm0,%xmm1 + +# qhasm: z14 = z3[2,2,2,2] +# asm 1: pshufd $0xaa,z14=int6464#3 +# asm 2: pshufd $0xaa,z14=%xmm2 +pshufd $0xaa,%xmm0,%xmm2 + +# qhasm: z3 = z3[3,3,3,3] +# asm 1: pshufd $0xff,z3=int6464#1 +# asm 2: pshufd $0xff,z3=%xmm0 +pshufd $0xff,%xmm0,%xmm0 + +# qhasm: orig4 = z4 +# asm 1: movdqa orig4=stack128#16 +# asm 2: movdqa orig4=240(%rsp) +movdqa %xmm1,240(%rsp) + +# qhasm: orig14 = z14 +# asm 1: movdqa orig14=stack128#17 +# asm 2: movdqa orig14=256(%rsp) +movdqa %xmm2,256(%rsp) + +# qhasm: orig3 = z3 +# asm 1: movdqa orig3=stack128#18 +# asm 2: movdqa orig3=272(%rsp) +movdqa %xmm0,272(%rsp) + +# qhasm: bytesatleast256: +._bytesatleast256: + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#3d +# asm 2: movl in8=%edx +movl 16(%rsp),%edx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#4d +# asm 2: movl 4+in9=%ecx +movl 4+32(%rsp),%ecx + +# qhasm: ((uint32 *) &orig8)[0] = in8 +# asm 1: movl orig8=stack128#19 +# asm 2: movl orig8=288(%rsp) +movl %edx,288(%rsp) + +# qhasm: ((uint32 *) &orig9)[0] = in9 +# asm 1: movl orig9=stack128#20 +# asm 2: movl orig9=304(%rsp) +movl %ecx,304(%rsp) + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,in9=int64#4 +# asm 2: mov in9=%rcx +mov %rdx,%rcx + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %edx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#3 +# asm 2: mov $12,>i=%rdx +mov $12,%rdx + +# qhasm: z5 = orig5 +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 64(%rsp),%xmm0 + +# qhasm: z10 = orig10 +# asm 1: movdqa z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 80(%rsp),%xmm1 + +# qhasm: z15 = orig15 +# asm 1: movdqa z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 96(%rsp),%xmm2 + +# qhasm: z14 = orig14 +# asm 1: movdqa z14=int6464#4 +# asm 2: movdqa z14=%xmm3 +movdqa 256(%rsp),%xmm3 + +# qhasm: z3 = orig3 +# asm 1: movdqa z3=int6464#5 +# asm 2: movdqa z3=%xmm4 +movdqa 272(%rsp),%xmm4 + +# qhasm: z6 = orig6 +# asm 1: movdqa z6=int6464#6 +# asm 2: movdqa z6=%xmm5 +movdqa 128(%rsp),%xmm5 + +# qhasm: z11 = orig11 +# asm 1: movdqa z11=int6464#7 +# asm 2: movdqa z11=%xmm6 +movdqa 144(%rsp),%xmm6 + +# qhasm: z1 = orig1 +# asm 1: movdqa z1=int6464#8 +# asm 2: movdqa z1=%xmm7 +movdqa 176(%rsp),%xmm7 + +# qhasm: z7 = orig7 +# asm 1: movdqa z7=int6464#9 +# asm 2: movdqa z7=%xmm8 +movdqa 192(%rsp),%xmm8 + +# qhasm: z13 = orig13 +# asm 1: movdqa z13=int6464#10 +# asm 2: movdqa z13=%xmm9 +movdqa 208(%rsp),%xmm9 + +# qhasm: z2 = orig2 +# asm 1: movdqa z2=int6464#11 +# asm 2: movdqa z2=%xmm10 +movdqa 224(%rsp),%xmm10 + +# qhasm: z9 = orig9 +# asm 1: movdqa z9=int6464#12 +# asm 2: movdqa z9=%xmm11 +movdqa 304(%rsp),%xmm11 + +# qhasm: z0 = orig0 +# asm 1: movdqa z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 112(%rsp),%xmm12 + +# qhasm: z12 = orig12 +# asm 1: movdqa z12=int6464#14 +# asm 2: movdqa z12=%xmm13 +movdqa 160(%rsp),%xmm13 + +# qhasm: z4 = orig4 +# asm 1: movdqa z4=int6464#15 +# asm 2: movdqa z4=%xmm14 +movdqa 240(%rsp),%xmm14 + +# qhasm: z8 = orig8 +# asm 1: movdqa z8=int6464#16 +# asm 2: movdqa z8=%xmm15 +movdqa 288(%rsp),%xmm15 + +# qhasm: mainloop1: +._mainloop1: + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y4 = z12 +# asm 1: movdqa y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm13,%xmm1 + +# qhasm: uint32323232 y4 += z0 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y9=int6464#2 +# asm 2: movdqa y9=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y9 += z5 +# asm 1: paddd r9=int6464#3 +# asm 2: movdqa r9=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y9 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#2 +# asm 2: movdqa y8=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y8 += z4 +# asm 1: paddd r8=int6464#3 +# asm 2: movdqa r8=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#2 +# asm 2: movdqa y13=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y13 += z9 +# asm 1: paddd r13=int6464#3 +# asm 2: movdqa r13=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y12=int6464#2 +# asm 2: movdqa y12=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y12 += z8 +# asm 1: paddd r12=int6464#3 +# asm 2: movdqa r12=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y12 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm11,%xmm1 + +# qhasm: uint32323232 y1 += z13 +# asm 1: paddd r1=int6464#3 +# asm 2: movdqa r1=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y1 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm15,%xmm1 + +# qhasm: uint32323232 y0 += z12 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm12,320(%rsp) + +# qhasm: y5 = z13 +# asm 1: movdqa y5=int6464#3 +# asm 2: movdqa y5=%xmm2 +movdqa %xmm9,%xmm2 + +# qhasm: uint32323232 y5 += z1 +# asm 1: paddd r5=int6464#13 +# asm 2: movdqa r5=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y14=int6464#3 +# asm 2: movdqa y14=%xmm2 +movdqa %xmm5,%xmm2 + +# qhasm: uint32323232 y14 += z10 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm2,%xmm12 + +# qhasm: uint32323232 y14 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm0,336(%rsp) + +# qhasm: y3 = z11 +# asm 1: movdqa y3=int6464#1 +# asm 2: movdqa y3=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y3 += z15 +# asm 1: paddd r3=int6464#13 +# asm 2: movdqa r3=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y3 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#1 +# asm 2: movdqa y2=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y2 += z14 +# asm 1: paddd r2=int6464#13 +# asm 2: movdqa r2=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#1 +# asm 2: movdqa y7=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y7 += z3 +# asm 1: paddd r7=int6464#13 +# asm 2: movdqa r7=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y6=int6464#1 +# asm 2: movdqa y6=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y6 += z2 +# asm 1: paddd r6=int6464#13 +# asm 2: movdqa r6=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm4,%xmm0 + +# qhasm: uint32323232 y11 += z7 +# asm 1: paddd r11=int6464#13 +# asm 2: movdqa r11=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y11 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm10,%xmm0 + +# qhasm: uint32323232 y10 += z6 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#1 +# asm 2: movdqa z0=%xmm0 +movdqa 320(%rsp),%xmm0 + +# qhasm: z10_stack = z10 +# asm 1: movdqa z10_stack=stack128#21 +# asm 2: movdqa z10_stack=320(%rsp) +movdqa %xmm1,320(%rsp) + +# qhasm: y1 = z3 +# asm 1: movdqa y1=int6464#2 +# asm 2: movdqa y1=%xmm1 +movdqa %xmm4,%xmm1 + +# qhasm: uint32323232 y1 += z0 +# asm 1: paddd r1=int6464#13 +# asm 2: movdqa r1=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y1 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y15=int6464#2 +# asm 2: movdqa y15=%xmm1 +movdqa %xmm8,%xmm1 + +# qhasm: uint32323232 y15 += z11 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm1,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z5=int6464#13 +# asm 2: movdqa z5=%xmm12 +movdqa 336(%rsp),%xmm12 + +# qhasm: z15_stack = z15 +# asm 1: movdqa z15_stack=stack128#22 +# asm 2: movdqa z15_stack=336(%rsp) +movdqa %xmm2,336(%rsp) + +# qhasm: y6 = z4 +# asm 1: movdqa y6=int6464#2 +# asm 2: movdqa y6=%xmm1 +movdqa %xmm14,%xmm1 + +# qhasm: uint32323232 y6 += z5 +# asm 1: paddd r6=int6464#3 +# asm 2: movdqa r6=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y6 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y2=int6464#2 +# asm 2: movdqa y2=%xmm1 +movdqa %xmm0,%xmm1 + +# qhasm: uint32323232 y2 += z1 +# asm 1: paddd r2=int6464#3 +# asm 2: movdqa r2=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y2 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y7=int6464#2 +# asm 2: movdqa y7=%xmm1 +movdqa %xmm12,%xmm1 + +# qhasm: uint32323232 y7 += z6 +# asm 1: paddd r7=int6464#3 +# asm 2: movdqa r7=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y7 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y3=int6464#2 +# asm 2: movdqa y3=%xmm1 +movdqa %xmm7,%xmm1 + +# qhasm: uint32323232 y3 += z2 +# asm 1: paddd r3=int6464#3 +# asm 2: movdqa r3=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y3 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y4=int6464#2 +# asm 2: movdqa y4=%xmm1 +movdqa %xmm5,%xmm1 + +# qhasm: uint32323232 y4 += z7 +# asm 1: paddd r4=int6464#3 +# asm 2: movdqa r4=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y4 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y0=int6464#2 +# asm 2: movdqa y0=%xmm1 +movdqa %xmm10,%xmm1 + +# qhasm: uint32323232 y0 += z3 +# asm 1: paddd r0=int6464#3 +# asm 2: movdqa r0=%xmm2 +movdqa %xmm1,%xmm2 + +# qhasm: uint32323232 y0 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z10=int6464#2 +# asm 2: movdqa z10=%xmm1 +movdqa 320(%rsp),%xmm1 + +# qhasm: z0_stack = z0 +# asm 1: movdqa z0_stack=stack128#21 +# asm 2: movdqa z0_stack=320(%rsp) +movdqa %xmm0,320(%rsp) + +# qhasm: y5 = z7 +# asm 1: movdqa y5=int6464#1 +# asm 2: movdqa y5=%xmm0 +movdqa %xmm8,%xmm0 + +# qhasm: uint32323232 y5 += z4 +# asm 1: paddd r5=int6464#3 +# asm 2: movdqa r5=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y5 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y11=int6464#1 +# asm 2: movdqa y11=%xmm0 +movdqa %xmm11,%xmm0 + +# qhasm: uint32323232 y11 += z10 +# asm 1: paddd r11=int6464#3 +# asm 2: movdqa r11=%xmm2 +movdqa %xmm0,%xmm2 + +# qhasm: uint32323232 y11 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,z15=int6464#3 +# asm 2: movdqa z15=%xmm2 +movdqa 336(%rsp),%xmm2 + +# qhasm: z5_stack = z5 +# asm 1: movdqa z5_stack=stack128#22 +# asm 2: movdqa z5_stack=336(%rsp) +movdqa %xmm12,336(%rsp) + +# qhasm: y12 = z14 +# asm 1: movdqa y12=int6464#1 +# asm 2: movdqa y12=%xmm0 +movdqa %xmm3,%xmm0 + +# qhasm: uint32323232 y12 += z15 +# asm 1: paddd r12=int6464#13 +# asm 2: movdqa r12=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y12 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,y8=int6464#1 +# asm 2: movdqa y8=%xmm0 +movdqa %xmm1,%xmm0 + +# qhasm: uint32323232 y8 += z11 +# asm 1: paddd r8=int6464#13 +# asm 2: movdqa r8=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y8 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y13=int6464#1 +# asm 2: movdqa y13=%xmm0 +movdqa %xmm2,%xmm0 + +# qhasm: uint32323232 y13 += z12 +# asm 1: paddd r13=int6464#13 +# asm 2: movdqa r13=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y13 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,y9=int6464#1 +# asm 2: movdqa y9=%xmm0 +movdqa %xmm6,%xmm0 + +# qhasm: uint32323232 y9 += z8 +# asm 1: paddd r9=int6464#13 +# asm 2: movdqa r9=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y9 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y14=int6464#1 +# asm 2: movdqa y14=%xmm0 +movdqa %xmm13,%xmm0 + +# qhasm: uint32323232 y14 += z13 +# asm 1: paddd r14=int6464#13 +# asm 2: movdqa r14=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y14 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,y10=int6464#1 +# asm 2: movdqa y10=%xmm0 +movdqa %xmm15,%xmm0 + +# qhasm: uint32323232 y10 += z9 +# asm 1: paddd r10=int6464#13 +# asm 2: movdqa r10=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y10 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,y15=int6464#1 +# asm 2: movdqa y15=%xmm0 +movdqa %xmm9,%xmm0 + +# qhasm: uint32323232 y15 += z14 +# asm 1: paddd r15=int6464#13 +# asm 2: movdqa r15=%xmm12 +movdqa %xmm0,%xmm12 + +# qhasm: uint32323232 y15 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,z0=int6464#13 +# asm 2: movdqa z0=%xmm12 +movdqa 320(%rsp),%xmm12 + +# qhasm: z5 = z5_stack +# asm 1: movdqa z5=int6464#1 +# asm 2: movdqa z5=%xmm0 +movdqa 336(%rsp),%xmm0 + +# qhasm: unsigned>? i -= 2 +# asm 1: sub $2, +ja ._mainloop1 + +# qhasm: uint32323232 z0 += orig0 +# asm 1: paddd in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: z0 <<<= 96 +# asm 1: pshufd $0x39,in0=int64#3 +# asm 2: movd in0=%rdx +movd %xmm12,%rdx + +# qhasm: in1 = z1 +# asm 1: movd in1=int64#4 +# asm 2: movd in1=%rcx +movd %xmm7,%rcx + +# qhasm: in2 = z2 +# asm 1: movd in2=int64#5 +# asm 2: movd in2=%r8 +movd %xmm10,%r8 + +# qhasm: in3 = z3 +# asm 1: movd in3=int64#6 +# asm 2: movd in3=%r9 +movd %xmm4,%r9 + +# qhasm: (uint32) in0 ^= *(uint32 *) (m + 192) +# asm 1: xorl 192(in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: z4 <<<= 96 +# asm 1: pshufd $0x39,in4=int64#3 +# asm 2: movd in4=%rdx +movd %xmm14,%rdx + +# qhasm: in5 = z5 +# asm 1: movd in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = z6 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm5,%r8 + +# qhasm: in7 = z7 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm8,%r9 + +# qhasm: (uint32) in4 ^= *(uint32 *) (m + 208) +# asm 1: xorl 208(in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: z8 <<<= 96 +# asm 1: pshufd $0x39,in8=int64#3 +# asm 2: movd in8=%rdx +movd %xmm15,%rdx + +# qhasm: in9 = z9 +# asm 1: movd in9=int64#4 +# asm 2: movd in9=%rcx +movd %xmm11,%rcx + +# qhasm: in10 = z10 +# asm 1: movd in10=int64#5 +# asm 2: movd in10=%r8 +movd %xmm1,%r8 + +# qhasm: in11 = z11 +# asm 1: movd in11=int64#6 +# asm 2: movd in11=%r9 +movd %xmm6,%r9 + +# qhasm: (uint32) in8 ^= *(uint32 *) (m + 224) +# asm 1: xorl 224(in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: z12 <<<= 96 +# asm 1: pshufd $0x39,in12=int64#3 +# asm 2: movd in12=%rdx +movd %xmm13,%rdx + +# qhasm: in13 = z13 +# asm 1: movd in13=int64#4 +# asm 2: movd in13=%rcx +movd %xmm9,%rcx + +# qhasm: in14 = z14 +# asm 1: movd in14=int64#5 +# asm 2: movd in14=%r8 +movd %xmm3,%r8 + +# qhasm: in15 = z15 +# asm 1: movd in15=int64#6 +# asm 2: movd in15=%r9 +movd %xmm2,%r9 + +# qhasm: (uint32) in12 ^= *(uint32 *) (m + 240) +# asm 1: xorl 240(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: bytes -= 256 +# asm 1: sub $256,? bytes - 0 +# asm 1: cmp $0, +jbe ._done +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesbetween1and255: +._bytesbetween1and255: + +# qhasm: unsignedctarget=int64#3 +# asm 2: mov ctarget=%rdx +mov %rdi,%rdx + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb + +# qhasm: out = &tmp +# asm 1: leaq out=int64#1 +# asm 2: leaq out=%rdi +leaq 416(%rsp),%rdi + +# qhasm: m = &tmp +# asm 1: leaq m=int64#2 +# asm 2: leaq m=%rsi +leaq 416(%rsp),%rsi +# comment:fp stack unchanged by fallthrough + +# qhasm: nocopy: +._nocopy: + +# qhasm: bytes_backup = bytes +# asm 1: movq bytes_backup=stack64#8 +# asm 2: movq bytes_backup=408(%rsp) +movq %r9,408(%rsp) + +# qhasm: diag0 = x0 +# asm 1: movdqa diag0=int6464#1 +# asm 2: movdqa diag0=%xmm0 +movdqa 48(%rsp),%xmm0 + +# qhasm: diag1 = x1 +# asm 1: movdqa diag1=int6464#2 +# asm 2: movdqa diag1=%xmm1 +movdqa 0(%rsp),%xmm1 + +# qhasm: diag2 = x2 +# asm 1: movdqa diag2=int6464#3 +# asm 2: movdqa diag2=%xmm2 +movdqa 16(%rsp),%xmm2 + +# qhasm: diag3 = x3 +# asm 1: movdqa diag3=int6464#4 +# asm 2: movdqa diag3=%xmm3 +movdqa 32(%rsp),%xmm3 + +# qhasm: a0 = diag1 +# asm 1: movdqa a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: i = 12 +# asm 1: mov $12,>i=int64#4 +# asm 2: mov $12,>i=%rcx +mov $12,%rcx + +# qhasm: mainloop2: +._mainloop2: + +# qhasm: uint32323232 a0 += diag0 +# asm 1: paddd a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a1=int6464#6 +# asm 2: movdqa a1=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b0 = a0 +# asm 1: movdqa b0=int6464#7 +# asm 2: movdqa b0=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a0 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a2=int6464#5 +# asm 2: movdqa a2=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b1 = a1 +# asm 1: movdqa b1=int6464#7 +# asm 2: movdqa b1=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a1 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a3=int6464#6 +# asm 2: movdqa a3=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b2 = a2 +# asm 1: movdqa b2=int6464#7 +# asm 2: movdqa b2=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a2 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,a4=int6464#5 +# asm 2: movdqa a4=%xmm4 +movdqa %xmm3,%xmm4 + +# qhasm: b3 = a3 +# asm 1: movdqa b3=int6464#7 +# asm 2: movdqa b3=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a3 <<= 18 +# asm 1: pslld $18,>= 14 +# asm 1: psrld $14,a5=int6464#6 +# asm 2: movdqa a5=%xmm5 +movdqa %xmm0,%xmm5 + +# qhasm: b4 = a4 +# asm 1: movdqa b4=int6464#7 +# asm 2: movdqa b4=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a4 <<= 7 +# asm 1: pslld $7,>= 25 +# asm 1: psrld $25,a6=int6464#5 +# asm 2: movdqa a6=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b5 = a5 +# asm 1: movdqa b5=int6464#7 +# asm 2: movdqa b5=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a5 <<= 9 +# asm 1: pslld $9,>= 23 +# asm 1: psrld $23,a7=int6464#6 +# asm 2: movdqa a7=%xmm5 +movdqa %xmm2,%xmm5 + +# qhasm: b6 = a6 +# asm 1: movdqa b6=int6464#7 +# asm 2: movdqa b6=%xmm6 +movdqa %xmm4,%xmm6 + +# qhasm: uint32323232 a6 <<= 13 +# asm 1: pslld $13,>= 19 +# asm 1: psrld $19,? i -= 4 +# asm 1: sub $4,a0=int6464#5 +# asm 2: movdqa a0=%xmm4 +movdqa %xmm1,%xmm4 + +# qhasm: b7 = a7 +# asm 1: movdqa b7=int6464#7 +# asm 2: movdqa b7=%xmm6 +movdqa %xmm5,%xmm6 + +# qhasm: uint32323232 a7 <<= 18 +# asm 1: pslld $18,b0=int6464#8,>b0=int6464#8 +# asm 2: pxor >b0=%xmm7,>b0=%xmm7 +pxor %xmm7,%xmm7 + +# qhasm: uint32323232 b7 >>= 14 +# asm 1: psrld $14, +ja ._mainloop2 + +# qhasm: uint32323232 diag0 += x0 +# asm 1: paddd in0=int64#4 +# asm 2: movd in0=%rcx +movd %xmm0,%rcx + +# qhasm: in12 = diag1 +# asm 1: movd in12=int64#5 +# asm 2: movd in12=%r8 +movd %xmm1,%r8 + +# qhasm: in8 = diag2 +# asm 1: movd in8=int64#6 +# asm 2: movd in8=%r9 +movd %xmm2,%r9 + +# qhasm: in4 = diag3 +# asm 1: movd in4=int64#7 +# asm 2: movd in4=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in5=int64#4 +# asm 2: movd in5=%rcx +movd %xmm0,%rcx + +# qhasm: in1 = diag1 +# asm 1: movd in1=int64#5 +# asm 2: movd in1=%r8 +movd %xmm1,%r8 + +# qhasm: in13 = diag2 +# asm 1: movd in13=int64#6 +# asm 2: movd in13=%r9 +movd %xmm2,%r9 + +# qhasm: in9 = diag3 +# asm 1: movd in9=int64#7 +# asm 2: movd in9=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in10=int64#4 +# asm 2: movd in10=%rcx +movd %xmm0,%rcx + +# qhasm: in6 = diag1 +# asm 1: movd in6=int64#5 +# asm 2: movd in6=%r8 +movd %xmm1,%r8 + +# qhasm: in2 = diag2 +# asm 1: movd in2=int64#6 +# asm 2: movd in2=%r9 +movd %xmm2,%r9 + +# qhasm: in14 = diag3 +# asm 1: movd in14=int64#7 +# asm 2: movd in14=%rax +movd %xmm3,%rax + +# qhasm: diag0 <<<= 96 +# asm 1: pshufd $0x39,in15=int64#4 +# asm 2: movd in15=%rcx +movd %xmm0,%rcx + +# qhasm: in11 = diag1 +# asm 1: movd in11=int64#5 +# asm 2: movd in11=%r8 +movd %xmm1,%r8 + +# qhasm: in7 = diag2 +# asm 1: movd in7=int64#6 +# asm 2: movd in7=%r9 +movd %xmm2,%r9 + +# qhasm: in3 = diag3 +# asm 1: movd in3=int64#7 +# asm 2: movd in3=%rax +movd %xmm3,%rax + +# qhasm: (uint32) in15 ^= *(uint32 *) (m + 60) +# asm 1: xorl 60(bytes=int64#6 +# asm 2: movq bytes=%r9 +movq 408(%rsp),%r9 + +# qhasm: in8 = ((uint32 *)&x2)[0] +# asm 1: movl in8=int64#4d +# asm 2: movl in8=%ecx +movl 16(%rsp),%ecx + +# qhasm: in9 = ((uint32 *)&x3)[1] +# asm 1: movl 4+in9=int64#5d +# asm 2: movl 4+in9=%r8d +movl 4+32(%rsp),%r8d + +# qhasm: in8 += 1 +# asm 1: add $1,in9=int64#5 +# asm 2: mov in9=%r8 +mov %rcx,%r8 + +# qhasm: (uint64) in9 >>= 32 +# asm 1: shr $32,x2=stack128#2 +# asm 2: movl x2=16(%rsp) +movl %ecx,16(%rsp) + +# qhasm: ((uint32 *)&x3)[1] = in9 +# asm 1: movl ? unsigned +ja ._bytesatleast65 +# comment:fp stack unchanged by jump + +# qhasm: goto bytesatleast64 if !unsigned< +jae ._bytesatleast64 + +# qhasm: m = out +# asm 1: mov m=int64#2 +# asm 2: mov m=%rsi +mov %rdi,%rsi + +# qhasm: out = ctarget +# asm 1: mov out=int64#1 +# asm 2: mov out=%rdi +mov %rdx,%rdi + +# qhasm: i = bytes +# asm 1: mov i=int64#4 +# asm 2: mov i=%rcx +mov %r9,%rcx + +# qhasm: while (i) { *out++ = *m++; --i } +rep movsb +# comment:fp stack unchanged by fallthrough + +# qhasm: bytesatleast64: +._bytesatleast64: +# comment:fp stack unchanged by fallthrough + +# qhasm: done: +._done: + +# qhasm: r11_caller = r11_stack +# asm 1: movq r11_caller=int64#9 +# asm 2: movq r11_caller=%r11 +movq 352(%rsp),%r11 + +# qhasm: r12_caller = r12_stack +# asm 1: movq r12_caller=int64#10 +# asm 2: movq r12_caller=%r12 +movq 360(%rsp),%r12 + +# qhasm: r13_caller = r13_stack +# asm 1: movq r13_caller=int64#11 +# asm 2: movq r13_caller=%r13 +movq 368(%rsp),%r13 + +# qhasm: r14_caller = r14_stack +# asm 1: movq r14_caller=int64#12 +# asm 2: movq r14_caller=%r14 +movq 376(%rsp),%r14 + +# qhasm: r15_caller = r15_stack +# asm 1: movq r15_caller=int64#13 +# asm 2: movq r15_caller=%r15 +movq 384(%rsp),%r15 + +# qhasm: rbx_caller = rbx_stack +# asm 1: movq rbx_caller=int64#14 +# asm 2: movq rbx_caller=%rbx +movq 392(%rsp),%rbx + +# qhasm: rbp_caller = rbp_stack +# asm 1: movq rbp_caller=int64#15 +# asm 2: movq rbp_caller=%rbp +movq 400(%rsp),%rbp + +# qhasm: leave +add %r11,%rsp +xor %rax,%rax +xor %rdx,%rdx +ret + +# qhasm: bytesatleast65: +._bytesatleast65: + +# qhasm: bytes -= 64 +# asm 1: sub $64,. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ /* diff --git a/zto/make-bsd.mk b/zto/make-bsd.mk index b038d13..39c6cef 100644 --- a/zto/make-bsd.mk +++ b/zto/make-bsd.mk @@ -1,3 +1,5 @@ +# This requires GNU make, which is typically "gmake" on BSD systems + INCLUDES= DEFS= LIBS= @@ -28,12 +30,14 @@ endif # Determine system build architecture from compiler target CC_MACH=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) -ZT_ARCHITECTURE=0 +ZT_ARCHITECTURE=999 ifeq ($(CC_MACH),x86_64) ZT_ARCHITECTURE=2 + ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),amd64) ZT_ARCHITECTURE=2 + ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),i386) ZT_ARCHITECTURE=1 @@ -43,16 +47,84 @@ ifeq ($(CC_MACH),i686) endif ifeq ($(CC_MACH),arm) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armel) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armhf) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv6) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv6zk) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv6kz) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv7) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),arm64) ZT_ARCHITECTURE=4 + override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),aarch64) ZT_ARCHITECTURE=4 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mipsel) + ZT_ARCHITECTURE=5 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips) + ZT_ARCHITECTURE=5 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips64) + ZT_ARCHITECTURE=6 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips64el) + ZT_ARCHITECTURE=6 + override DEFS+=-DZT_NO_TYPE_PUNNING endif -DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" -CXXFLAGS+=$(CFLAGS) -fno-rtti -std=c++11 -D_GLIBCXX_USE_C99 -D_GLIBCXX_USE_C99_MATH -D_GLIBCXX_USE_C99_MATH_TR1 +# Fail if system architecture could not be determined +ifeq ($(ZT_ARCHITECTURE),999) +ERR=$(error FATAL: architecture could not be determined from $(CC) -dumpmachine: $CC_MACH) +.PHONY: err +err: ; $(ERR) +endif + +# Build faster crypto on some targets +ifeq ($(ZT_USE_X64_ASM_SALSA2012),1) + override DEFS+=-DZT_USE_X64_ASM_SALSA2012 + override OBJS+=ext/x64-salsa2012-asm/salsa2012.o +endif +ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) + override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 + override OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o +endif + +override DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" + +CXXFLAGS+=$(CFLAGS) -fno-rtti -std=c++11 #-D_GLIBCXX_USE_C99 -D_GLIBCXX_USE_C99_MATH -D_GLIBCXX_USE_C99_MATH_TR1 all: one @@ -67,7 +139,7 @@ selftest: $(OBJS) selftest.o $(STRIP) zerotier-selftest clean: - rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* + rm -rf *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o build-* zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* $(OBJS) debug: FORCE make -j 4 ZT_DEBUG=1 diff --git a/zto/make-linux.mk b/zto/make-linux.mk index a606d58..5fb489b 100644 --- a/zto/make-linux.mk +++ b/zto/make-linux.mk @@ -8,7 +8,7 @@ ifeq ($(origin CXX),default) endif INCLUDES?= -DEFS?=-D_FORTIFY_SOURCE=2 +DEFS?= LDLIBS?= DESTDIR?= @@ -54,15 +54,16 @@ ifeq ($(ZT_RULES_ENGINE_DEBUGGING),1) endif ifeq ($(ZT_DEBUG),1) - DEFS+=-DZT_TRACE - override CFLAGS+=-Wall -g -O -pthread $(INCLUDES) $(DEFS) - override CXXFLAGS+=-Wall -g -O -std=c++11 -pthread $(INCLUDES) $(DEFS) + override DEFS+=-DZT_TRACE + override CFLAGS+=-Wall -g -pthread $(INCLUDES) $(DEFS) + override CXXFLAGS+=-Wall -g -std=c++11 -pthread $(INCLUDES) $(DEFS) override LDFLAGS+= STRIP?=echo # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! -node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS) +node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 -g -pthread $(INCLUDES) $(DEFS) else + override DEFS+=-D_FORTIFY_SOURCE=2 CFLAGS?=-O3 -fstack-protector override CFLAGS+=-Wall -fPIE -pthread $(INCLUDES) -DNDEBUG $(DEFS) CXXFLAGS?=-O3 -fstack-protector @@ -72,6 +73,10 @@ else STRIP+=--strip-all endif +ifeq ($(ZT_USE_TEST_TAP),1) + override DEFS+=-DZT_USE_TEST_TAP +endif + # Uncomment for gprof profile build #CFLAGS=-Wall -g -pg -pthread $(INCLUDES) $(DEFS) #CXXFLAGS=-Wall -g -pg -pthread $(INCLUDES) $(DEFS) @@ -80,52 +85,121 @@ endif # Determine system build architecture from compiler target CC_MACH=$(shell $(CC) -dumpmachine | cut -d '-' -f 1) -ZT_ARCHITECTURE=0 +ZT_ARCHITECTURE=999 ifeq ($(CC_MACH),x86_64) ZT_ARCHITECTURE=2 + ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),amd64) ZT_ARCHITECTURE=2 + ZT_USE_X64_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),i386) ZT_ARCHITECTURE=1 endif +ifeq ($(CC_MACH),i486) + ZT_ARCHITECTURE=1 +endif +ifeq ($(CC_MACH),i586) + ZT_ARCHITECTURE=1 +endif ifeq ($(CC_MACH),i686) ZT_ARCHITECTURE=1 endif ifeq ($(CC_MACH),arm) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armel) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armhf) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv6) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv6zk) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 +endif +ifeq ($(CC_MACH),armv6kz) + ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),armv7) ZT_ARCHITECTURE=3 + override DEFS+=-DZT_NO_TYPE_PUNNING + ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),arm64) ZT_ARCHITECTURE=4 + override DEFS+=-DZT_NO_TYPE_PUNNING endif ifeq ($(CC_MACH),aarch64) ZT_ARCHITECTURE=4 + override DEFS+=-DZT_NO_TYPE_PUNNING endif -DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" - -# Define some conservative CPU instruction set flags for arm32 since there's a ton of variation out there -ifeq ($(ZT_ARCHITECTURE),3) - override CFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp - override CXXFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfloat-abi=hard -mfpu=vfp +ifeq ($(CC_MACH),mipsel) + ZT_ARCHITECTURE=5 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips) + ZT_ARCHITECTURE=5 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips64) + ZT_ARCHITECTURE=6 + override DEFS+=-DZT_NO_TYPE_PUNNING +endif +ifeq ($(CC_MACH),mips64el) + ZT_ARCHITECTURE=6 override DEFS+=-DZT_NO_TYPE_PUNNING endif -# Define this to build a static binary, which is needed to make this runnable on a few ancient Linux distros +# Fail if system architecture could not be determined +ifeq ($(ZT_ARCHITECTURE),999) +ERR=$(error FATAL: architecture could not be determined from $(CC) -dumpmachine: $CC_MACH) +.PHONY: err +err: ; $(ERR) +endif + +# Disable software updates by default on Linux since that is normally done with package management +override DEFS+=-DZT_BUILD_PLATFORM=1 -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" + +# Static builds, which are currently done for a number of Linux targets ifeq ($(ZT_STATIC),1) override LDFLAGS+=-static + ifeq ($(ZT_ARCHITECTURE),3) + ifeq ($(shell if [ -e /usr/bin/dpkg ]; then dpkg --print-architecture; fi),armel) + override CFLAGS+=-march=armv5te -mfloat-abi=soft -msoft-float -mno-unaligned-access -marm + override CXXFLAGS+=-march=armv5te -mfloat-abi=soft -msoft-float -mno-unaligned-access -marm + ZT_USE_ARM32_NEON_ASM_SALSA2012=0 + else + override CFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -mno-unaligned-access -marm + override CXXFLAGS+=-march=armv6zk -mcpu=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -mno-unaligned-access -marm + ZT_USE_ARM32_NEON_ASM_SALSA2012=0 + endif + endif +endif + +# Build faster crypto on some targets +ifeq ($(ZT_USE_X64_ASM_SALSA2012),1) + override DEFS+=-DZT_USE_X64_ASM_SALSA2012 + override OBJS+=ext/x64-salsa2012-asm/salsa2012.o +endif +ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) + override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 + override OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o endif all: one diff --git a/zto/make-mac.mk b/zto/make-mac.mk index 8ff1b77..6676f45 100644 --- a/zto/make-mac.mk +++ b/zto/make-mac.mk @@ -37,6 +37,10 @@ ifeq ($(ZT_ENABLE_CLUSTER),1) DEFS+=-DZT_ENABLE_CLUSTER endif +# Use fast ASM Salsa20/12 for x64 processors +DEFS+=-DZT_USE_X64_ASM_SALSA2012 +OBJS+=ext/x64-salsa2012-asm/salsa2012.o + # Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o @@ -59,6 +63,9 @@ CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ all: one macui +ext/x64-salsa2012-asm/salsa2012.o: + $(CC) $(CFLAGS) -c ext/x64-salsa2012-asm/salsa2012.s -o ext/x64-salsa2012-asm/salsa2012.o + one: $(OBJS) service/OneService.o one.o $(CXX) $(CXXFLAGS) -o zerotier-one $(OBJS) service/OneService.o one.o $(LIBS) $(STRIP) zerotier-one diff --git a/zto/node/Address.hpp b/zto/node/Address.hpp index 4a5883b..9d2d173 100644 --- a/zto/node/Address.hpp +++ b/zto/node/Address.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ADDRESS_HPP diff --git a/zto/node/Array.hpp b/zto/node/Array.hpp index 19b29eb..5c61647 100644 --- a/zto/node/Array.hpp +++ b/zto/node/Array.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ARRAY_HPP diff --git a/zto/node/AtomicCounter.hpp b/zto/node/AtomicCounter.hpp index a0f29ba..e1864db 100644 --- a/zto/node/AtomicCounter.hpp +++ b/zto/node/AtomicCounter.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ATOMICCOUNTER_HPP diff --git a/zto/node/Buffer.hpp b/zto/node/Buffer.hpp index 37f39e7..ae242c7 100644 --- a/zto/node/Buffer.hpp +++ b/zto/node/Buffer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BUFFER_HPP diff --git a/zto/node/C25519.cpp b/zto/node/C25519.cpp index e9ffecc..a78e046 100644 --- a/zto/node/C25519.cpp +++ b/zto/node/C25519.cpp @@ -1,5 +1,3 @@ -// Code taken from NaCl by D. J. Bernstein and others - /* Matthew Dempsky Public domain. @@ -7,7 +5,7 @@ Derived from public domain code by D. J. Bernstein. */ // Modified very slightly for ZeroTier One by Adam Ierymenko -// (no functional changes) +// This code remains in the public domain. #include #include diff --git a/zto/node/C25519.hpp b/zto/node/C25519.hpp index b19d969..da9ba66 100644 --- a/zto/node/C25519.hpp +++ b/zto/node/C25519.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_C25519_HPP diff --git a/zto/node/Capability.cpp b/zto/node/Capability.cpp index c178e56..0e02025 100644 --- a/zto/node/Capability.cpp +++ b/zto/node/Capability.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Capability.hpp" diff --git a/zto/node/Capability.hpp b/zto/node/Capability.hpp index 5ef6c99..8d4b908 100644 --- a/zto/node/Capability.hpp +++ b/zto/node/Capability.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CAPABILITY_HPP @@ -24,6 +32,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -58,9 +67,11 @@ class RuntimeEnvironment; * handed off between nodes. Limited transferrability of capabilities is * a feature of true capability based security. */ -class Capability +class Capability : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_CAPABILITY; } + Capability() { memset(this,0,sizeof(Capability)); diff --git a/zto/node/CertificateOfMembership.cpp b/zto/node/CertificateOfMembership.cpp index 9bf7021..a5445e4 100644 --- a/zto/node/CertificateOfMembership.cpp +++ b/zto/node/CertificateOfMembership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "CertificateOfMembership.hpp" diff --git a/zto/node/CertificateOfMembership.hpp b/zto/node/CertificateOfMembership.hpp index ae976b5..739d539 100644 --- a/zto/node/CertificateOfMembership.hpp +++ b/zto/node/CertificateOfMembership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFMEMBERSHIP_HPP @@ -27,6 +35,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "Buffer.hpp" #include "Address.hpp" #include "C25519.hpp" @@ -68,9 +77,11 @@ class RuntimeEnvironment; * This is a memcpy()'able structure and is safe (in a crash sense) to modify * without locks. */ -class CertificateOfMembership +class CertificateOfMembership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COM; } + /** * Reserved qualifier IDs * @@ -155,18 +166,23 @@ public: /** * @return True if there's something here */ - inline operator bool() const throw() { return (_qualifierCount != 0); } + inline operator bool() const { return (_qualifierCount != 0); } + + /** + * @return Credential ID, always 0 for COMs + */ + inline uint32_t id() const { return 0; } /** * @return Timestamp for this cert and maximum delta for timestamp */ - inline std::pair timestamp() const + inline uint64_t timestamp() const { for(unsigned int i=0;i<_qualifierCount;++i) { if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP) - return std::pair(_qualifiers[i].value,_qualifiers[i].maxDelta); + return _qualifiers[i].value; } - return std::pair(0ULL,0ULL); + return 0; } /** @@ -258,12 +274,12 @@ public: /** * @return True if signed */ - inline bool isSigned() const throw() { return (_signedBy); } + inline bool isSigned() const { return (_signedBy); } /** * @return Address that signed this certificate or null address if none */ - inline const Address &signedBy() const throw() { return _signedBy; } + inline const Address &signedBy() const { return _signedBy; } template inline void serialize(Buffer &b) const @@ -321,7 +337,6 @@ public: } inline bool operator==(const CertificateOfMembership &c) const - throw() { if (_signedBy != c._signedBy) return false; @@ -335,7 +350,7 @@ public: } return (_signature == c._signature); } - inline bool operator!=(const CertificateOfMembership &c) const throw() { return (!(*this == c)); } + inline bool operator!=(const CertificateOfMembership &c) const { return (!(*this == c)); } private: struct _Qualifier diff --git a/zto/node/CertificateOfOwnership.cpp b/zto/node/CertificateOfOwnership.cpp index 2bd181e..31d0ae1 100644 --- a/zto/node/CertificateOfOwnership.cpp +++ b/zto/node/CertificateOfOwnership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "CertificateOfOwnership.hpp" diff --git a/zto/node/CertificateOfOwnership.hpp b/zto/node/CertificateOfOwnership.hpp index 8c47582..95039a2 100644 --- a/zto/node/CertificateOfOwnership.hpp +++ b/zto/node/CertificateOfOwnership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFOWNERSHIP_HPP @@ -25,6 +33,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -45,9 +54,11 @@ class RuntimeEnvironment; /** * Certificate indicating ownership of a network identifier */ -class CertificateOfOwnership +class CertificateOfOwnership : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COO; } + enum Thing { THING_NULL = 0, @@ -69,6 +80,8 @@ public: _thingCount(0), _issuedTo(issuedTo) { + memset(_thingTypes,0,sizeof(_thingTypes)); + memset(_thingValues,0,sizeof(_thingValues)); } inline uint64_t networkId() const { return _networkId; } diff --git a/zto/node/CertificateOfRepresentation.hpp b/zto/node/CertificateOfRepresentation.hpp index 02e961c..92a71bc 100644 --- a/zto/node/CertificateOfRepresentation.hpp +++ b/zto/node/CertificateOfRepresentation.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,12 +14,21 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CERTIFICATEOFREPRESENTATION_HPP #define ZT_CERTIFICATEOFREPRESENTATION_HPP #include "Constants.hpp" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Identity.hpp" @@ -47,14 +56,17 @@ namespace ZeroTier { * roots can shield nodes entirely and p2p connectivity behind them can * be disabled. This will be desirable for a number of use cases. */ -class CertificateOfRepresentation +class CertificateOfRepresentation : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_COR; } + CertificateOfRepresentation() { memset(this,0,sizeof(CertificateOfRepresentation)); } + inline uint32_t id() const { return 0; } inline uint64_t timestamp() const { return _timestamp; } inline const Address &representative(const unsigned int i) const { return _reps[i]; } inline unsigned int repCount() const { return _repCount; } diff --git a/zto/node/Cluster.cpp b/zto/node/Cluster.cpp index 54206f9..4d2dea7 100644 --- a/zto/node/Cluster.cpp +++ b/zto/node/Cluster.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_ENABLE_CLUSTER diff --git a/zto/node/Cluster.hpp b/zto/node/Cluster.hpp index 08e32a9..74b091f 100644 --- a/zto/node/Cluster.hpp +++ b/zto/node/Cluster.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTER_HPP diff --git a/zto/node/Constants.hpp b/zto/node/Constants.hpp index 410a245..d3c8749 100644 --- a/zto/node/Constants.hpp +++ b/zto/node/Constants.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CONSTANTS_HPP @@ -289,7 +297,7 @@ #define ZT_PEER_PING_PERIOD 60000 /** - * Paths are considered expired if they have not produced a real packet in this long + * Paths are considered expired if they have not sent us a real packet in this long */ #define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 4) + 3000) diff --git a/zto/node/Credential.hpp b/zto/node/Credential.hpp new file mode 100644 index 0000000..bc81919 --- /dev/null +++ b/zto/node/Credential.hpp @@ -0,0 +1,66 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_CREDENTIAL_HPP +#define ZT_CREDENTIAL_HPP + +#include +#include +#include + +#include +#include +#include +#include + +#include "Constants.hpp" + +namespace ZeroTier { + +/** + * Base class for credentials + */ +class Credential +{ +public: + /** + * Do not change type code IDs -- these are used in Revocation objects and elsewhere + */ + enum Type + { + CREDENTIAL_TYPE_NULL = 0, + CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership + CREDENTIAL_TYPE_CAPABILITY = 2, + CREDENTIAL_TYPE_TAG = 3, + CREDENTIAL_TYPE_COO = 4, // CertificateOfOwnership + CREDENTIAL_TYPE_COR = 5, // CertificateOfRepresentation + CREDENTIAL_TYPE_REVOCATION = 6 + }; +}; + +} // namespace ZeroTier + +#endif diff --git a/zto/node/Dictionary.hpp b/zto/node/Dictionary.hpp index fa9e288..4413d62 100644 --- a/zto/node/Dictionary.hpp +++ b/zto/node/Dictionary.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_DICTIONARY_HPP @@ -91,6 +99,8 @@ public: return *this; } + inline operator bool() const { return (_d[0] != 0); } + /** * Load a dictionary from a C-string * @@ -176,12 +186,12 @@ public: j = 0; esc = false; ++p; - while ((*p != 0)&&(*p != '\r')&&(*p != '\n')) { + while ((*p != 0)&&(*p != 13)&&(*p != 10)) { if (esc) { esc = false; switch(*p) { - case 'r': dest[j++] = '\r'; break; - case 'n': dest[j++] = '\n'; break; + case 'r': dest[j++] = 13; break; + case 'n': dest[j++] = 10; break; case '0': dest[j++] = (char)0; break; case 'e': dest[j++] = '='; break; default: dest[j++] = *p; break; @@ -207,7 +217,7 @@ public: dest[j] = (char)0; return j; } else { - while ((*p)&&(*p != '\r')&&(*p != '\n')) { + while ((*p)&&(*p != 13)&&(*p != 10)) { if (++p == eof) { dest[0] = (char)0; return -1; @@ -299,7 +309,7 @@ public: unsigned int j = i; if (j > 0) { - _d[j++] = '\n'; + _d[j++] = (char)10; if (j == C) { _d[i] = (char)0; return false; @@ -326,8 +336,8 @@ public: while ( ((vlen < 0)&&(*p)) || (k < vlen) ) { switch(*p) { case 0: - case '\r': - case '\n': + case 13: + case 10: case '\\': case '=': _d[j++] = '\\'; @@ -337,8 +347,8 @@ public: } switch(*p) { case 0: _d[j++] = '0'; break; - case '\r': _d[j++] = 'r'; break; - case '\n': _d[j++] = 'n'; break; + case 13: _d[j++] = 'r'; break; + case 10: _d[j++] = 'n'; break; case '\\': _d[j++] = '\\'; break; case '=': _d[j++] = 'e'; break; } @@ -416,46 +426,6 @@ public: return (this->get(key,tmp,2) >= 0); } - /** - * Erase a key from this dictionary - * - * Use this before add() to ensure that a key is replaced if it might - * already be present. - * - * @param key Key to erase - * @return True if key was found and erased - */ - inline bool erase(const char *key) - { - char d2[C]; - char *saveptr = (char *)0; - unsigned int d2ptr = 0; - bool found = false; - for(char *f=Utils::stok(_d,"\r\n",&saveptr);(f);f=Utils::stok((char *)0,"\r\n",&saveptr)) { - if (*f) { - const char *p = f; - const char *k = key; - while ((*k)&&(*p)) { - if (*k != *p) - break; - ++k; - ++p; - } - if (*k) { - p = f; - while (*p) - d2[d2ptr++] = *(p++); - d2[d2ptr++] = '\n'; - } else { - found = true; - } - } - } - d2[d2ptr++] = (char)0; - memcpy(_d,d2,d2ptr); - return found; - } - /** * @return Value of C template parameter */ diff --git a/zto/node/Hashtable.hpp b/zto/node/Hashtable.hpp index 66f2990..c46ed68 100644 --- a/zto/node/Hashtable.hpp +++ b/zto/node/Hashtable.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_HASHTABLE_HPP diff --git a/zto/node/Identity.cpp b/zto/node/Identity.cpp index 89fdb83..ba77aa4 100644 --- a/zto/node/Identity.cpp +++ b/zto/node/Identity.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -45,7 +53,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub // ordinary Salsa20 is randomly seekable. This is good for a cipher // but is not what we want for sequential memory-harndess. memset(genmem,0,ZT_IDENTITY_GEN_MEMORY); - Salsa20 s20(digest,256,(char *)digest + 32); + Salsa20 s20(digest,(char *)digest + 32); s20.crypt20((char *)genmem,(char *)genmem,64); for(unsigned long i=64;i. + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_IDENTITY_HPP diff --git a/zto/node/IncomingPacket.cpp b/zto/node/IncomingPacket.cpp index 52794fd..126da53 100644 --- a/zto/node/IncomingPacket.cpp +++ b/zto/node/IncomingPacket.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -1200,16 +1208,14 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt switch(addrType) { case 4: { - InetAddress a(field(ptr,4),4,at(ptr + 4)); - - bool redundant = false; - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimal(a); - } else { - redundant = peer->hasActivePathTo(now,a); - } - - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { + const InetAddress a(field(ptr,4),4,at(ptr + 4)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); @@ -1219,16 +1225,14 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt } } break; case 6: { - InetAddress a(field(ptr,16),16,at(ptr + 16)); - - bool redundant = false; - if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) { - peer->setClusterOptimal(a); - } else { - redundant = peer->hasActivePathTo(now,a); - } - - if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) { + const InetAddress a(field(ptr,16),16,at(ptr + 16)); + if ( + ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && // not being told to forget + (!( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) == 0) && (peer->hasActivePathTo(now,a)) )) && // not already known + (RR->node->shouldUsePathForZeroTierTraffic(tPtr,peer->address(),_path->localAddress(),a)) ) // should use path + { + if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) + peer->setClusterPreferred(a); if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) { TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str()); peer->attemptToContactAt(tPtr,InetAddress(),a,now,false,0); diff --git a/zto/node/IncomingPacket.hpp b/zto/node/IncomingPacket.hpp index 3d4a2e0..43a1ea1 100644 --- a/zto/node/IncomingPacket.hpp +++ b/zto/node/IncomingPacket.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_INCOMINGPACKET_HPP diff --git a/zto/node/InetAddress.cpp b/zto/node/InetAddress.cpp index 3dc0122..0fbb2d6 100644 --- a/zto/node/InetAddress.cpp +++ b/zto/node/InetAddress.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -302,7 +310,7 @@ InetAddress InetAddress::network() const return false; } #endif - + bool InetAddress::containsAddress(const InetAddress &addr) const { if (addr.ss_family == ss_family) { diff --git a/zto/node/InetAddress.hpp b/zto/node/InetAddress.hpp index 0533ea0..4cb9a4d 100644 --- a/zto/node/InetAddress.hpp +++ b/zto/node/InetAddress.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_INETADDRESS_HPP @@ -347,16 +355,16 @@ struct InetAddress : public sockaddr_storage */ InetAddress network() const; -#ifdef ZT_SDK - /** +#ifdef ZT_SDK + /** * Test whether this IPv6 prefix matches the prefix of a given IPv6 address * * @param addr Address to check * @return True if this IPv6 prefix matches the prefix of a given IPv6 address */ - bool isEqualPrefix(const InetAddress &addr) const; + bool isEqualPrefix(const InetAddress &addr) const; #endif - + /** * Test whether this IP/netmask contains this address * diff --git a/zto/node/MAC.hpp b/zto/node/MAC.hpp index 95623f1..e7717d9 100644 --- a/zto/node/MAC.hpp +++ b/zto/node/MAC.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MAC_HPP diff --git a/zto/node/Membership.cpp b/zto/node/Membership.cpp index 22c13c8..466f902 100644 --- a/zto/node/Membership.cpp +++ b/zto/node/Membership.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -33,11 +41,13 @@ namespace ZeroTier { Membership::Membership() : _lastUpdatedMulticast(0), _lastPushedCom(0), - _comRevocationThreshold(0) + _comRevocationThreshold(0), + _revocations(4), + _remoteTags(4), + _remoteCaps(4), + _remoteCoos(4) { - for(unsigned int i=0;i= 0) { sendCap = &(nconf.capabilities[localCapabilityIndex]); - if ( (_localCaps[localCapabilityIndex].id != sendCap->id()) || ((now - _localCaps[localCapabilityIndex].lastPushed) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCaps[localCapabilityIndex].lastPushed = now; - _localCaps[localCapabilityIndex].id = sendCap->id(); - } else sendCap = (const Capability *)0; + if ( ((now - _localCredLastPushed.cap[localCapabilityIndex]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) + _localCredLastPushed.cap[localCapabilityIndex] = now; + else sendCap = (const Capability *)0; } else sendCap = (const Capability *)0; const Tag *sendTags[ZT_MAX_NETWORK_TAGS]; unsigned int sendTagCount = 0; for(unsigned int t=0;t= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localTags[t].lastPushed = now; - _localTags[t].id = nconf.tags[t].id(); + if ( ((now - _localCredLastPushed.tag[t]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.tag[t] = now; sendTags[sendTagCount++] = &(nconf.tags[t]); } } @@ -66,9 +74,8 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u const CertificateOfOwnership *sendCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; unsigned int sendCooCount = 0; for(unsigned int c=0;c= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { - _localCoos[c].lastPushed = now; - _localCoos[c].id = nconf.certificatesOfOwnership[c].id(); + if ( ((now - _localCredLastPushed.coo[c]) >= ZT_CREDENTIAL_PUSH_EVERY) || (force) ) { + _localCredLastPushed.coo[c] = now; sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]); } } @@ -117,21 +124,15 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const u } } -const Tag *Membership::getTag(const NetworkConfig &nconf,const uint32_t id) const -{ - const _RemoteCredential *const *t = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)id,_RemoteCredentialComp()); - return ( ((t != &(_remoteTags[ZT_MAX_NETWORK_CAPABILITIES]))&&((*t)->id == (uint64_t)id)) ? ((((*t)->lastReceived)&&(_isCredentialTimestampValid(nconf,**t))) ? &((*t)->credential) : (const Tag *)0) : (const Tag *)0); -} - Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfMembership &com) { - const uint64_t newts = com.timestamp().first; + const uint64_t newts = com.timestamp(); if (newts <= _comRevocationThreshold) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (revoked)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; } - const uint64_t oldts = _com.timestamp().first; + const uint64_t oldts = _com.timestamp(); if (newts < oldts) { TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (older than current)",com.issuedTo().toString().c_str(),com.networkId()); return ADD_REJECTED; @@ -154,84 +155,73 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) +// Template out addCredential() for many cred types to avoid copypasta +template +static Membership::AddCredentialResult _addCredImpl(Hashtable &remoteCreds,const Hashtable &revocations,const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const C &cred) { - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)tag.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)tag.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > tag.timestamp()) ) { - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (revoked or too old)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_REJECTED; + C *rc = remoteCreds.get(cred.id()); + if (rc) { + if (rc->timestamp() > cred.timestamp()) { + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (older than credential we have)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; } - if (have->credential == tag) { - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (redundant)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_ACCEPTED_REDUNDANT; + if (*rc == cred) { + //TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (redundant)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_ACCEPTED_REDUNDANT; } } - switch(tag.verify(RR,tPtr)) { + const uint64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id())); + if ((rt)&&(*rt >= cred.timestamp())) { + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (timestamp below revocation threshold)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; + } + + switch(cred.verify(RR,tPtr)) { default: - TRACE("addCredential(Tag) for %s on %.16llx REJECTED (invalid)",tag.issuedTo().toString().c_str(),tag.networkId()); - return ADD_REJECTED; + TRACE("addCredential(type==%d) for %s on %.16llx REJECTED (invalid)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + return Membership::ADD_REJECTED; case 0: - TRACE("addCredential(Tag) for %s on %.16llx ACCEPTED (new)",tag.issuedTo().toString().c_str(),tag.networkId()); - if (!have) have = _newTag(tag.id()); - have->lastReceived = RR->node->now(); - have->credential = tag; - return ADD_ACCEPTED_NEW; + TRACE("addCredential(type==%d) for %s on %.16llx ACCEPTED (new)",(int)C::credentialType(),cred.issuedTo().toString().c_str(),cred.networkId()); + if (!rc) + rc = &(remoteCreds[cred.id()]); + *rc = cred; + return Membership::ADD_ACCEPTED_NEW; case 1: - return ADD_DEFERRED_FOR_WHOIS; + return Membership::ADD_DEFERRED_FOR_WHOIS; } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)cap.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)cap.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > cap.timestamp()) ) { - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (revoked or too old)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_REJECTED; - } - if (have->credential == cap) { - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (redundant)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_ACCEPTED_REDUNDANT; - } - } - - switch(cap.verify(RR,tPtr)) { - default: - TRACE("addCredential(Capability) for %s on %.16llx REJECTED (invalid)",cap.issuedTo().toString().c_str(),cap.networkId()); - return ADD_REJECTED; - case 0: - TRACE("addCredential(Capability) for %s on %.16llx ACCEPTED (new)",cap.issuedTo().toString().c_str(),cap.networkId()); - if (!have) have = _newCapability(cap.id()); - have->lastReceived = RR->node->now(); - have->credential = cap; - return ADD_ACCEPTED_NEW; - case 1: - return ADD_DEFERRED_FOR_WHOIS; - } -} +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Tag &tag) { return _addCredImpl(_remoteTags,_revocations,RR,tPtr,nconf,tag); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Capability &cap) { return _addCredImpl(_remoteCaps,_revocations,RR,tPtr,nconf,cap); } +Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) { return _addCredImpl(_remoteCoos,_revocations,RR,tPtr,nconf,coo); } Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev) { + uint64_t *rt; switch(rev.verify(RR,tPtr)) { default: return ADD_REJECTED; case 0: { - const uint64_t now = RR->node->now(); - switch(rev.type()) { + const Credential::Type ct = rev.type(); + switch(ct) { + case Credential::CREDENTIAL_TYPE_COM: + if (rev.threshold() > _comRevocationThreshold) { + _comRevocationThreshold = rev.threshold(); + return ADD_ACCEPTED_NEW; + } + return ADD_ACCEPTED_REDUNDANT; + case Credential::CREDENTIAL_TYPE_CAPABILITY: + case Credential::CREDENTIAL_TYPE_TAG: + case Credential::CREDENTIAL_TYPE_COO: + rt = &(_revocations[credentialKey(ct,rev.credentialId())]); + if (*rt < rev.threshold()) { + *rt = rev.threshold(); + return ADD_ACCEPTED_NEW; + } + return ADD_ACCEPTED_REDUNDANT; default: return ADD_REJECTED; - case Revocation::CREDENTIAL_TYPE_COM: - return (_revokeCom(rev) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_CAPABILITY: - return (_revokeCap(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_TAG: - return (_revokeTag(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); - case Revocation::CREDENTIAL_TYPE_COO: - return (_revokeCoo(rev,now) ? ADD_ACCEPTED_NEW : ADD_ACCEPTED_REDUNDANT); } } case 1: @@ -239,157 +229,11 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme } } -Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo) +void Membership::clean(const uint64_t now,const NetworkConfig &nconf) { - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)coo.id(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)coo.id())) ? *htmp : (_RemoteCredential *)0; - if (have) { - if ( (!_isCredentialTimestampValid(nconf,*have)) || (have->credential.timestamp() > coo.timestamp()) ) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (revoked or too old)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_REJECTED; - } - if (have->credential == coo) { - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (redundant)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_ACCEPTED_REDUNDANT; - } - } - - switch(coo.verify(RR,tPtr)) { - default: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx REJECTED (invalid)",coo.issuedTo().toString().c_str(),coo.networkId()); - return ADD_REJECTED; - case 0: - TRACE("addCredential(CertificateOfOwnership) for %s on %.16llx ACCEPTED (new)",coo.issuedTo().toString().c_str(),coo.networkId()); - if (!have) have = _newCoo(coo.id()); - have->lastReceived = RR->node->now(); - have->credential = coo; - return ADD_ACCEPTED_NEW; - case 1: - return ADD_DEFERRED_FOR_WHOIS; - } -} - -Membership::_RemoteCredential *Membership::_newTag(const uint64_t id) -{ - _RemoteCredential *t = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - t = _remoteTags[i]; - break; - } else if (_remoteTags[i]->lastReceived <= minlr) { - t = _remoteTags[i]; - minlr = _remoteTags[i]->lastReceived; - } - } - - if (t) { - t->id = id; - t->lastReceived = 0; - t->revocationThreshold = 0; - t->credential = Tag(); - } - - std::sort(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),_RemoteCredentialComp()); - return t; -} - -Membership::_RemoteCredential *Membership::_newCapability(const uint64_t id) -{ - _RemoteCredential *c = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - c = _remoteCaps[i]; - break; - } else if (_remoteCaps[i]->lastReceived <= minlr) { - c = _remoteCaps[i]; - minlr = _remoteCaps[i]->lastReceived; - } - } - - if (c) { - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->credential = Capability(); - } - - std::sort(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),_RemoteCredentialComp()); - return c; -} - -Membership::_RemoteCredential *Membership::_newCoo(const uint64_t id) -{ - _RemoteCredential *c = NULL; - uint64_t minlr = 0xffffffffffffffffULL; - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) { - c = _remoteCoos[i]; - break; - } else if (_remoteCoos[i]->lastReceived <= minlr) { - c = _remoteCoos[i]; - minlr = _remoteCoos[i]->lastReceived; - } - } - - if (c) { - c->id = id; - c->lastReceived = 0; - c->revocationThreshold = 0; - c->credential = CertificateOfOwnership(); - } - - std::sort(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),_RemoteCredentialComp()); - return c; -} - -bool Membership::_revokeCom(const Revocation &rev) -{ - if (rev.threshold() > _comRevocationThreshold) { - _comRevocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeCap(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCaps[0]),&(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newCapability(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeTag(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteTags[0]),&(_remoteTags[ZT_MAX_NETWORK_TAGS]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newTag(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; -} - -bool Membership::_revokeCoo(const Revocation &rev,const uint64_t now) -{ - _RemoteCredential *const *htmp = std::lower_bound(&(_remoteCoos[0]),&(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]),(uint64_t)rev.credentialId(),_RemoteCredentialComp()); - _RemoteCredential *have = ((htmp != &(_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]))&&((*htmp)->id == (uint64_t)rev.credentialId())) ? *htmp : (_RemoteCredential *)0; - if (!have) have = _newCoo(rev.credentialId()); - if (rev.threshold() > have->revocationThreshold) { - have->lastReceived = now; - have->revocationThreshold = rev.threshold(); - return true; - } - return false; + _cleanCredImpl(nconf,_remoteTags); + _cleanCredImpl(nconf,_remoteCaps); + _cleanCredImpl(nconf,_remoteCoos); } } // namespace ZeroTier diff --git a/zto/node/Membership.hpp b/zto/node/Membership.hpp index c28d598..5e4475d 100644 --- a/zto/node/Membership.hpp +++ b/zto/node/Membership.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MEMBERSHIP_HPP @@ -23,6 +31,8 @@ #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Credential.hpp" +#include "Hashtable.hpp" #include "CertificateOfMembership.hpp" #include "Capability.hpp" #include "Tag.hpp" @@ -45,35 +55,6 @@ class Network; */ class Membership { -private: - template - struct _RemoteCredential - { - _RemoteCredential() : id(ZT_MEMBERSHIP_CRED_ID_UNUSED),lastReceived(0),revocationThreshold(0) {} - uint64_t id; - uint64_t lastReceived; // last time we got this credential - uint64_t revocationThreshold; // credentials before this time are invalid - T credential; - inline bool operator<(const _RemoteCredential &c) const { return (id < c.id); } - }; - - template - struct _RemoteCredentialComp - { - inline bool operator()(const _RemoteCredential *a,const _RemoteCredential *b) const { return (a->id < b->id); } - inline bool operator()(const uint64_t a,const _RemoteCredential *b) const { return (a < b->id); } - inline bool operator()(const _RemoteCredential *a,const uint64_t b) const { return (a->id < b); } - inline bool operator()(const uint64_t a,const uint64_t b) const { return (a < b); } - }; - - // Used to track push state for network config tags[] and capabilities[] entries - struct _LocalCredentialPushState - { - _LocalCredentialPushState() : lastPushed(0),id(0) {} - uint64_t lastPushed; // last time we sent our own copy of this credential - uint64_t id; - }; - public: enum AddCredentialResult { @@ -83,72 +64,6 @@ public: ADD_DEFERRED_FOR_WHOIS }; - /** - * Iterator to scan forward through capabilities in ascending order of ID - */ - class CapabilityIterator - { - public: - CapabilityIterator(const Membership &m,const NetworkConfig &nconf) : - _m(&m), - _c(&nconf), - _i(&(m._remoteCaps[0])) {} - - inline const Capability *next() - { - for(;;) { - if ((_i != &(_m->_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Capability *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,**_i)) { - ++_i; - return tmp; - } else ++_i; - } else { - return (const Capability *)0; - } - } - } - - private: - const Membership *_m; - const NetworkConfig *_c; - const _RemoteCredential *const *_i; - }; - friend class CapabilityIterator; - - /** - * Iterator to scan forward through tags in ascending order of ID - */ - class TagIterator - { - public: - TagIterator(const Membership &m,const NetworkConfig &nconf) : - _m(&m), - _c(&nconf), - _i(&(m._remoteTags[0])) {} - - inline const Tag *next() - { - for(;;) { - if ((_i != &(_m->_remoteTags[ZT_MAX_NETWORK_TAGS]))&&((*_i)->id != ZT_MEMBERSHIP_CRED_ID_UNUSED)) { - const Tag *tmp = &((*_i)->credential); - if (_m->_isCredentialTimestampValid(*_c,**_i)) { - ++_i; - return tmp; - } else ++_i; - } else { - return (const Tag *)0; - } - } - } - - private: - const Membership *_m; - const NetworkConfig *_c; - const _RemoteCredential *const *_i; - }; - friend class TagIterator; - Membership(); /** @@ -168,19 +83,19 @@ public: void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex,const bool force); /** - * Check whether we should push MULTICAST_LIKEs to this peer + * Check whether we should push MULTICAST_LIKEs to this peer, and update last sent time if true * * @param now Current time * @return True if we should update multicasts */ - inline bool shouldLikeMulticasts(const uint64_t now) const { return ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD); } - - /** - * Set time we last updated multicasts for this peer - * - * @param now Current time - */ - inline void likingMulticasts(const uint64_t now) { _lastUpdatedMulticast = now; } + inline bool multicastLikeGate(const uint64_t now) + { + if ((now - _lastUpdatedMulticast) >= ZT_MULTICAST_ANNOUNCE_PERIOD) { + _lastUpdatedMulticast = now; + return true; + } + return false; + } /** * Check whether the peer represented by this Membership should be allowed on this network at all @@ -190,10 +105,8 @@ public: */ inline bool isAllowedOnNetwork(const NetworkConfig &nconf) const { - if (nconf.isPublic()) - return true; - if (_com.timestamp().first <= _comRevocationThreshold) - return false; + if (nconf.isPublic()) return true; + if (_com.timestamp() <= _comRevocationThreshold) return false; return nconf.com.agreesWith(_com); } @@ -208,21 +121,28 @@ public: template inline bool hasCertificateOfOwnershipFor(const NetworkConfig &nconf,const T &r) const { - for(unsigned int i=0;iid == ZT_MEMBERSHIP_CRED_ID_UNUSED) - break; - if ((_isCredentialTimestampValid(nconf,*_remoteCoos[i]))&&(_remoteCoos[i]->credential.owns(r))) + uint32_t *k = (uint32_t *)0; + CertificateOfOwnership *v = (CertificateOfOwnership *)0; + Hashtable< uint32_t,CertificateOfOwnership >::Iterator i(*(const_cast< Hashtable< uint32_t,CertificateOfOwnership> *>(&_remoteCoos))); + while (i.next(k,v)) { + if (_isCredentialTimestampValid(nconf,*v)&&(v->owns(r))) return true; } return false; } /** + * Get a remote member's tag (if we have it) + * * @param nconf Network configuration * @param id Tag ID * @return Pointer to tag or NULL if not found */ - const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const; + inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const + { + const Tag *const t = _remoteTags.get(id); + return (((t)&&(_isCredentialTimestampValid(nconf,*t))) ? t : (Tag *)0); + } /** * Validate and add a credential if signature is okay and it's otherwise good @@ -242,29 +162,59 @@ public: /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); /** * Validate and add a credential if signature is okay and it's otherwise good */ - AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const CertificateOfOwnership &coo); + AddCredentialResult addCredential(const RuntimeEnvironment *RR,void *tPtr,const NetworkConfig &nconf,const Revocation &rev); + + /** + * Clean internal databases of stale entries + * + * @param now Current time + * @param nconf Current network configuration + */ + void clean(const uint64_t now,const NetworkConfig &nconf); + + /** + * Reset last pushed time for local credentials + * + * This is done when we update our network configuration and our credentials have changed + */ + inline void resetPushState() + { + _lastPushedCom = 0; + memset(&_localCredLastPushed,0,sizeof(_localCredLastPushed)); + } + + /** + * Generates a key for the internal use in indexing credentials by type and credential ID + */ + static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); } private: - _RemoteCredential *_newTag(const uint64_t id); - _RemoteCredential *_newCapability(const uint64_t id); - _RemoteCredential *_newCoo(const uint64_t id); - bool _revokeCom(const Revocation &rev); - bool _revokeCap(const Revocation &rev,const uint64_t now); - bool _revokeTag(const Revocation &rev,const uint64_t now); - bool _revokeCoo(const Revocation &rev,const uint64_t now); + template + inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &remoteCredential) const + { + const uint64_t ts = remoteCredential.timestamp(); + if (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) { + const uint64_t *threshold = _revocations.get(credentialKey(C::credentialType(),remoteCredential.id())); + return ((!threshold)||(ts > *threshold)); + } + return false; + } template - inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const _RemoteCredential &remoteCredential) const + void _cleanCredImpl(const NetworkConfig &nconf,Hashtable &remoteCreds) { - if (!remoteCredential.lastReceived) - return false; - const uint64_t ts = remoteCredential.credential.timestamp(); - return ( (((ts >= nconf.timestamp) ? (ts - nconf.timestamp) : (nconf.timestamp - ts)) <= nconf.credentialTimeMaxDelta) && (ts > remoteCredential.revocationThreshold) ); + uint32_t *k = (uint32_t *)0; + C *v = (C *)0; + typename Hashtable::Iterator i(remoteCreds); + while (i.next(k,v)) { + if (!_isCredentialTimestampValid(nconf,*v)) + remoteCreds.erase(*k); + } } // Last time we pushed MULTICAST_LIKE(s) @@ -279,20 +229,50 @@ private: // Remote member's latest network COM CertificateOfMembership _com; - // Sorted (in ascending order of ID) arrays of pointers to remote credentials - _RemoteCredential *_remoteTags[ZT_MAX_NETWORK_TAGS]; - _RemoteCredential *_remoteCaps[ZT_MAX_NETWORK_CAPABILITIES]; - _RemoteCredential *_remoteCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Revocations by credentialKey() + Hashtable< uint64_t,uint64_t > _revocations; - // This is the RAM allocated for remote credential cache objects - _RemoteCredential _tagMem[ZT_MAX_NETWORK_TAGS]; - _RemoteCredential _capMem[ZT_MAX_NETWORK_CAPABILITIES]; - _RemoteCredential _cooMem[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Remote credentials that we have received from this member (and that are valid) + Hashtable< uint32_t,Tag > _remoteTags; + Hashtable< uint32_t,Capability > _remoteCaps; + Hashtable< uint32_t,CertificateOfOwnership > _remoteCoos; - // Local credential push state tracking - _LocalCredentialPushState _localTags[ZT_MAX_NETWORK_TAGS]; - _LocalCredentialPushState _localCaps[ZT_MAX_NETWORK_CAPABILITIES]; - _LocalCredentialPushState _localCoos[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + // Time we last pushed our local credentials to this member + struct { + uint64_t tag[ZT_MAX_NETWORK_TAGS]; + uint64_t cap[ZT_MAX_NETWORK_CAPABILITIES]; + uint64_t coo[ZT_MAX_CERTIFICATES_OF_OWNERSHIP]; + } _localCredLastPushed; + +public: + class CapabilityIterator + { + public: + CapabilityIterator(Membership &m,const NetworkConfig &nconf) : + _hti(m._remoteCaps), + _k((uint32_t *)0), + _c((Capability *)0), + _m(m), + _nconf(nconf) + { + } + + inline Capability *next() + { + while (_hti.next(_k,_c)) { + if (_m._isCredentialTimestampValid(_nconf,*_c)) + return _c; + } + return (Capability *)0; + } + + private: + Hashtable< uint32_t,Capability >::Iterator _hti; + uint32_t *_k; + Capability *_c; + Membership &_m; + const NetworkConfig &_nconf; + }; }; } // namespace ZeroTier diff --git a/zto/node/MulticastGroup.hpp b/zto/node/MulticastGroup.hpp index be4e808..4240db6 100644 --- a/zto/node/MulticastGroup.hpp +++ b/zto/node/MulticastGroup.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MULTICASTGROUP_HPP diff --git a/zto/node/Multicaster.cpp b/zto/node/Multicaster.cpp index 8e534b5..5221336 100644 --- a/zto/node/Multicaster.cpp +++ b/zto/node/Multicaster.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/node/Multicaster.hpp b/zto/node/Multicaster.hpp index f646a5b..2186e9c 100644 --- a/zto/node/Multicaster.hpp +++ b/zto/node/Multicaster.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MULTICASTER_HPP diff --git a/zto/node/Mutex.hpp b/zto/node/Mutex.hpp index d451ede..6f1d347 100644 --- a/zto/node/Mutex.hpp +++ b/zto/node/Mutex.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_MUTEX_HPP diff --git a/zto/node/Network.cpp b/zto/node/Network.cpp index 0abfdf8..ee0f861 100644 --- a/zto/node/Network.cpp +++ b/zto/node/Network.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -534,9 +542,9 @@ static _doZtFilterResult _doZtFilter( } if (inbound) { if (membership) { - if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) + if ((src)&&(membership->hasCertificateOfOwnershipFor(nconf,src))) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_IP_AUTHENTICATED; - if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) + if (membership->hasCertificateOfOwnershipFor(nconf,macSource)) ownershipVerificationMask |= ZT_RULE_PACKET_CHARACTERISTICS_SENDER_MAC_AUTHENTICATED; } } else { @@ -729,7 +737,8 @@ Network::~Network() char n[128]; if (_destroyed) { - RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + // This is done in Node::leave() so we can pass tPtr + //RR->node->configureVirtualNetworkPort((void *)0,_id,&_uPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id); RR->node->dataStoreDelete((void *)0,n); } else { @@ -993,6 +1002,9 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg) uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Address &source,const Buffer &chunk,unsigned int ptr) { + if (_destroyed) + return 0; + const unsigned int start = ptr; ptr += 8; // skip network ID, which is already obviously known @@ -1140,24 +1152,37 @@ uint64_t Network::handleConfigChunk(void *tPtr,const uint64_t packetId,const Add int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToDisk) { + if (_destroyed) + return 0; + // _lock is NOT locked when this is called try { if ((nconf.issuedTo != RR->identity.address())||(nconf.networkId != _id)) - return 0; + return 0; // invalid config that is not for us or not for this network if (_config == nconf) return 1; // OK config, but duplicate of what we already have ZT_VirtualNetworkConfig ctmp; bool oldPortInitialized; - { + { // do things that require lock here, but unlock before calling callbacks Mutex::Lock _l(_lock); + _config = nconf; _lastConfigUpdate = RR->node->now(); _netconfFailure = NETCONF_FAILURE_NONE; + oldPortInitialized = _portInitialized; _portInitialized = true; + _externalConfig(&ctmp); + + Address *a = (Address *)0; + Membership *m = (Membership *)0; + Hashtable::Iterator i(_memberships); + while (i.next(a,m)) + m->resetPushState(); } + _portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp); if (saveToDisk) { @@ -1180,6 +1205,9 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD void Network::requestConfiguration(void *tPtr) { + if (_destroyed) + return; + /* ZeroTier addresses can't begin with 0xff, so this is used to mark controllerless * network IDs. Controllerless network IDs only support unicast IPv6 using the 6plane * addressing scheme and have the following format: 0xffSSSSEEEE000000 where SSSS @@ -1299,10 +1327,9 @@ bool Network::gate(void *tPtr,const SharedPtr &peer) if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) { if (!m) m = &(_membership(peer->address())); - if (m->shouldLikeMulticasts(now)) { + if (m->multicastLikeGate(now)) { m->pushCredentials(RR,tPtr,now,peer->address(),_config,-1,false); _announceMulticastGroupsTo(tPtr,peer->address(),_allMulticastGroups()); - m->likingMulticasts(now); } return true; } @@ -1338,6 +1365,7 @@ void Network::clean() while (i.next(a,m)) { if (!RR->topology->getPeerNoCache(*a)) _memberships.erase(*a); + else m->clean(now,_config); } } } @@ -1546,8 +1574,7 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu } // Make sure that all "network anchors" have Membership records so we will - // push multicasts to them. Note that _membership() also does this but in a - // piecemeal on-demand fashion. + // push multicasts to them. const std::vector
anchors(_config.anchors()); for(std::vector
::const_iterator a(anchors.begin());a!=anchors.end();++a) _membership(*a); @@ -1559,11 +1586,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu Hashtable::Iterator i(_memberships); while (i.next(a,m)) { m->pushCredentials(RR,tPtr,now,*a,_config,-1,false); - if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) { - if (!newMulticastGroup) - m->likingMulticasts(now); + if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) ) _announceMulticastGroupsTo(tPtr,*a,groups); - } } } } diff --git a/zto/node/Network.hpp b/zto/node/Network.hpp index fccc267..cce6c41 100644 --- a/zto/node/Network.hpp +++ b/zto/node/Network.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORK_HPP @@ -344,9 +352,8 @@ public: /** * Destroy this network * - * This causes the network to disable itself, destroy its tap device, and on - * delete to delete all trace of itself on disk and remove any persistent tap - * device instances. Call this when a network is being removed from the system. + * This sets the network to completely remove itself on delete. This also prevents the + * call of the normal port shutdown event on delete. */ void destroy(); @@ -364,7 +371,7 @@ public: /** * @return Externally usable pointer-to-pointer exported via the core API */ - inline void **userPtr() throw() { return &_uPtr; } + inline void **userPtr() { return &_uPtr; } private: ZT_VirtualNetworkStatus _status() const; diff --git a/zto/node/NetworkConfig.cpp b/zto/node/NetworkConfig.cpp index fe7393e..9effe52 100644 --- a/zto/node/NetworkConfig.cpp +++ b/zto/node/NetworkConfig.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/node/NetworkConfig.hpp b/zto/node/NetworkConfig.hpp index 85c2409..7bae6a9 100644 --- a/zto/node/NetworkConfig.hpp +++ b/zto/node/NetworkConfig.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORKCONFIG_HPP diff --git a/zto/node/NetworkController.hpp b/zto/node/NetworkController.hpp index 0634f43..63d44a4 100644 --- a/zto/node/NetworkController.hpp +++ b/zto/node/NetworkController.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NETWORKCONFIGMASTER_HPP diff --git a/zto/node/Node.cpp b/zto/node/Node.cpp index e7dc637..5848d95 100644 --- a/zto/node/Node.cpp +++ b/zto/node/Node.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -50,7 +58,6 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 _RR(this), RR(&_RR), _uPtr(uptr), - _prngStreamPtr(0), _now(now), _lastPingCheck(0), _lastHousekeepingRun(0) @@ -59,19 +66,14 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6 throw std::runtime_error("callbacks struct version mismatch"); memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks)); + Utils::getSecureRandom((void *)_prngState,sizeof(_prngState)); + _online = false; memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr)); memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo)); memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification)); - // Use Salsa20 alone as a high-quality non-crypto PRNG - char foo[32]; - Utils::getSecureRandom(foo,32); - _prng.init(foo,256,foo); - memset(_prngStream,0,sizeof(_prngStream)); - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - std::string idtmp(dataStoreGet(tptr,"identity.secret")); if ((!idtmp.length())||(!RR->identity.fromString(idtmp))||(!RR->identity.hasPrivate())) { TRACE("identity.secret not found, generating..."); @@ -305,26 +307,35 @@ ZT_ResultCode Node::join(uint64_t nwid,void *uptr,void *tptr) { Mutex::Lock _l(_networks_m); SharedPtr nw = _network(nwid); - if(!nw) - _networks.push_back(std::pair< uint64_t,SharedPtr >(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr)))); - std::sort(_networks.begin(),_networks.end()); // will sort by nwid since it's the first in a pair<> + if(!nw) { + const std::pair< uint64_t,SharedPtr > nn(nwid,SharedPtr(new Network(RR,tptr,nwid,uptr))); + _networks.insert(std::upper_bound(_networks.begin(),_networks.end(),nn),nn); + } return ZT_RESULT_OK; } ZT_ResultCode Node::leave(uint64_t nwid,void **uptr,void *tptr) { + ZT_VirtualNetworkConfig ctmp; std::vector< std::pair< uint64_t,SharedPtr > > newn; + void **nUserPtr = (void **)0; Mutex::Lock _l(_networks_m); + for(std::vector< std::pair< uint64_t,SharedPtr > >::const_iterator n(_networks.begin());n!=_networks.end();++n) { - if (n->first != nwid) + if (n->first != nwid) { newn.push_back(*n); - else { + } else { if (uptr) - *uptr = n->second->userPtr(); + *uptr = *n->second->userPtr(); n->second->destroy(); + nUserPtr = n->second->userPtr(); } } _networks.swap(newn); + + if (nUserPtr) + RR->node->configureVirtualNetworkPort(tptr,nwid,nUserPtr,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp); + return ZT_RESULT_OK; } @@ -398,17 +409,17 @@ ZT_PeerList *Node::peers() const p->latency = pi->second->latency(); p->role = RR->topology->role(pi->second->identity().address()); - std::vector< std::pair< SharedPtr,bool > > paths(pi->second->paths(_now)); + std::vector< SharedPtr > paths(pi->second->paths(_now)); SharedPtr bestp(pi->second->getBestPath(_now,false)); p->pathCount = 0; - for(std::vector< std::pair< SharedPtr,bool > >::iterator path(paths.begin());path!=paths.end();++path) { - memcpy(&(p->paths[p->pathCount].address),&(path->first->address()),sizeof(struct sockaddr_storage)); - p->paths[p->pathCount].lastSend = path->first->lastOut(); - p->paths[p->pathCount].lastReceive = path->first->lastIn(); - p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->first->address()); - p->paths[p->pathCount].linkQuality = (int)path->first->linkQuality(); - p->paths[p->pathCount].expired = path->second; - p->paths[p->pathCount].preferred = (path->first == bestp) ? 1 : 0; + for(std::vector< SharedPtr >::iterator path(paths.begin());path!=paths.end();++path) { + memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); + p->paths[p->pathCount].lastSend = (*path)->lastOut(); + p->paths[p->pathCount].lastReceive = (*path)->lastIn(); + p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); + p->paths[p->pathCount].linkQuality = (int)(*path)->linkQuality(); + p->paths[p->pathCount].expired = 0; + p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; ++p->pathCount; } } @@ -487,7 +498,8 @@ int Node::sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *d void Node::setNetconfMaster(void *networkControllerInstance) { RR->localNetworkController = reinterpret_cast(networkControllerInstance); - RR->localNetworkController->init(RR->identity,this); + if (networkControllerInstance) + RR->localNetworkController->init(RR->identity,this); } ZT_ResultCode Node::circuitTestBegin(void *tptr,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)) @@ -692,10 +704,14 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...) uint64_t Node::prng() { - unsigned int p = (++_prngStreamPtr % ZT_NODE_PRNG_BUF_SIZE); - if (!p) - _prng.crypt12(_prngStream,_prngStream,sizeof(_prngStream)); - return _prngStream[p]; + // https://en.wikipedia.org/wiki/Xorshift#xorshift.2B + uint64_t x = _prngState[0]; + const uint64_t y = _prngState[1]; + _prngState[0] = y; + x ^= x << 23; + const uint64_t z = x ^ y ^ (x >> 17) ^ (y >> 26); + _prngState[1] = z; + return z + y; } void Node::postCircuitTestReport(const ZT_CircuitTestReport *report) diff --git a/zto/node/Node.hpp b/zto/node/Node.hpp index 03bd7a8..9558716 100644 --- a/zto/node/Node.hpp +++ b/zto/node/Node.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NODE_HPP @@ -50,9 +58,6 @@ #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255 #define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31 -// Size of PRNG stream buffer -#define ZT_NODE_PRNG_BUF_SIZE 64 - namespace ZeroTier { class World; @@ -312,13 +317,10 @@ private: Mutex _backgroundTasksLock; - unsigned int _prngStreamPtr; - Salsa20 _prng; - uint64_t _prngStream[ZT_NODE_PRNG_BUF_SIZE]; // repeatedly encrypted with _prng to yield a high-quality non-crypto PRNG stream - uint64_t _now; uint64_t _lastPingCheck; uint64_t _lastHousekeepingRun; + volatile uint64_t _prngState[2]; bool _online; }; diff --git a/zto/node/NonCopyable.hpp b/zto/node/NonCopyable.hpp index 6d4daa8..25c71b1 100644 --- a/zto/node/NonCopyable.hpp +++ b/zto/node/NonCopyable.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NONCOPYABLE_HPP__ diff --git a/zto/node/OutboundMulticast.cpp b/zto/node/OutboundMulticast.cpp index 285bfa5..a2341ff 100644 --- a/zto/node/OutboundMulticast.cpp +++ b/zto/node/OutboundMulticast.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Constants.hpp" diff --git a/zto/node/OutboundMulticast.hpp b/zto/node/OutboundMulticast.hpp index 0ecf113..0c98880 100644 --- a/zto/node/OutboundMulticast.hpp +++ b/zto/node/OutboundMulticast.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OUTBOUNDMULTICAST_HPP diff --git a/zto/node/Packet.cpp b/zto/node/Packet.cpp index 756f314..d60a3a3 100644 --- a/zto/node/Packet.cpp +++ b/zto/node/Packet.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -24,6 +32,13 @@ #include "Packet.hpp" +#ifdef ZT_USE_X64_ASM_SALSA2012 +#include "../ext/x64-salsa2012-asm/salsa2012.h" +#endif +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +#include "../ext/arm32-neon-salsa2012-asm/salsa2012.h" +#endif + #ifdef _MSC_VER #define FORCE_INLINE static __forceinline #include @@ -36,6 +51,34 @@ namespace ZeroTier { /************************************************************************** */ + +/* Set up macros for fast single-pass ASM Salsa20/12 crypto, if we have it */ + +// x64 SSE crypto +#ifdef ZT_USE_X64_ASM_SALSA2012 +#define ZT_HAS_FAST_CRYPTO() (true) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_amd64_xmm6(reinterpret_cast(b),(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// ARM (32-bit) NEON crypto (must be detected) +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +class _FastCryptoChecker +{ +public: + _FastCryptoChecker() : canHas(zt_arm_has_neon()) {} + bool canHas; +}; +static const _FastCryptoChecker _ZT_FAST_CRYPTO_CHECK; +#define ZT_HAS_FAST_CRYPTO() (_ZT_FAST_CRYPTO_CHECK.canHas) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) zt_salsa2012_armneon3_xor(reinterpret_cast(b),(const unsigned char *)0,(l),reinterpret_cast(n),reinterpret_cast(k)) +#endif + +// No fast crypto available +#ifndef ZT_HAS_FAST_CRYPTO +#define ZT_HAS_FAST_CRYPTO() (false) +#define ZT_FAST_SINGLE_PASS_SALSA2012(b,l,n,k) {} +#endif + /************************************************************************** */ /* LZ4 is shipped encapsulated into Packet in an anonymous namespace. @@ -1064,7 +1107,7 @@ const char *Packet::errorString(ErrorCode e) void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); // Mask least significant 3 bits of packet ID with counter to embed packet send counter for QoS use @@ -1074,23 +1117,35 @@ void Packet::armor(const void *key,bool encryptPayload,unsigned int counter) setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE); _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,data + ZT_PACKET_IDX_IV); - - // MAC key is always the first 32 bytes of the Salsa20 key stream - // This is the same construction DJB's NaCl library uses - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - - uint8_t *const payload = data + ZT_PACKET_IDX_VERB; - const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; - if (encryptPayload) - s20.crypt12(payload,payload,payloadLen); - Poly1305::compute(mac,payload,payloadLen,macKey); - memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + if (ZT_HAS_FAST_CRYPTO()) { + const unsigned int encryptLen = (encryptPayload) ? (size() - ZT_PACKET_IDX_VERB) : 0; + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,encryptLen + 64,(data + ZT_PACKET_IDX_IV),mangledKey); + Salsa20::memxor(data + ZT_PACKET_IDX_VERB,reinterpret_cast(keyStream + 8),encryptLen); + uint64_t mac[2]; + Poly1305::compute(mac,data + ZT_PACKET_IDX_VERB,size() - ZT_PACKET_IDX_VERB,keyStream); +#ifdef ZT_NO_TYPE_PUNNING + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); +#else + (*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) = mac[0]; +#endif + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint8_t *const payload = data + ZT_PACKET_IDX_VERB; + const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; + if (encryptPayload) + s20.crypt12(payload,payload,payloadLen); + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); + memcpy(data + ZT_PACKET_IDX_MAC,mac,8); + } } bool Packet::dearmor(const void *key) { - uint8_t mangledKey[32],macKey[32],mac[16]; + uint8_t mangledKey[32]; uint8_t *const data = reinterpret_cast(unsafeData()); const unsigned int payloadLen = size() - ZT_PACKET_IDX_VERB; unsigned char *const payload = data + ZT_PACKET_IDX_VERB; @@ -1098,15 +1153,36 @@ bool Packet::dearmor(const void *key) if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) { _salsa20MangleKey((const unsigned char *)key,mangledKey); - Salsa20 s20(mangledKey,256,data + ZT_PACKET_IDX_IV); - - s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); - Poly1305::compute(mac,payload,payloadLen,macKey); - if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) - return false; // MAC failed, packet is corrupt, modified, or is not from the sender - - if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) - s20.crypt12(payload,payload,payloadLen); + if (ZT_HAS_FAST_CRYPTO()) { + uint64_t keyStream[(ZT_PROTO_MAX_PACKET_LENGTH + 64 + 8) / 8]; + ZT_FAST_SINGLE_PASS_SALSA2012(keyStream,((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) ? (payloadLen + 64) : 64),(data + ZT_PACKET_IDX_IV),mangledKey); + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,keyStream); +#ifdef ZT_NO_TYPE_PUNNING + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; +#else + if ((*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) != mac[0]) // also secure, constant time + return false; +#endif + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) + Salsa20::memxor(data + ZT_PACKET_IDX_VERB,reinterpret_cast(keyStream + 8),payloadLen); + } else { + Salsa20 s20(mangledKey,data + ZT_PACKET_IDX_IV); + uint64_t macKey[4]; + s20.crypt12(ZERO_KEY,macKey,sizeof(macKey)); + uint64_t mac[2]; + Poly1305::compute(mac,payload,payloadLen,macKey); +#ifdef ZT_NO_TYPE_PUNNING + if (!Utils::secureEq(mac,data + ZT_PACKET_IDX_MAC,8)) + return false; +#else + if ((*reinterpret_cast(data + ZT_PACKET_IDX_MAC)) != mac[0]) // also secure, constant time + return false; +#endif + if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012) + s20.crypt12(payload,payload,payloadLen); + } return true; } else { @@ -1120,7 +1196,7 @@ void Packet::cryptField(const void *key,unsigned int start,unsigned int len) uint8_t iv[8]; for(int i=0;i<8;++i) iv[i] = data[i]; iv[7] &= 0xf8; // mask off least significant 3 bits of packet ID / IV since this is unset when this function gets called - Salsa20 s20(key,256,iv); + Salsa20 s20(key,iv); s20.crypt12(data + start,data + start,len); } diff --git a/zto/node/Packet.hpp b/zto/node/Packet.hpp index 8ad2c0f..1de679e 100644 --- a/zto/node/Packet.hpp +++ b/zto/node/Packet.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_N_PACKET_HPP diff --git a/zto/node/Path.cpp b/zto/node/Path.cpp index 7366b56..a5fe1aa 100644 --- a/zto/node/Path.cpp +++ b/zto/node/Path.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Path.hpp" diff --git a/zto/node/Path.hpp b/zto/node/Path.hpp index aef628d..32bceae 100644 --- a/zto/node/Path.hpp +++ b/zto/node/Path.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PATH_HPP diff --git a/zto/node/Peer.cpp b/zto/node/Peer.cpp index 0795a6e..0190583 100644 --- a/zto/node/Peer.cpp +++ b/zto/node/Peer.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "../version.h" @@ -27,14 +35,6 @@ #include "Cluster.hpp" #include "Packet.hpp" -#ifndef AF_MAX -#if AF_INET > AF_INET6 -#define AF_MAX AF_INET -#else -#define AF_MAX AF_INET6 -#endif -#endif - namespace ZeroTier { Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) : @@ -51,18 +51,15 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastComRequestSent(0), _lastCredentialsReceived(0), _lastTrustEstablishedPacketReceived(0), - _remoteClusterOptimal4(0), _vProto(0), _vMajor(0), _vMinor(0), _vRevision(0), _id(peerIdentity), - _numPaths(0), _latency(0), _directPathPushCutoffCount(0), _credentialsCutoffCount(0) { - memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6)); if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw std::runtime_error("new peer identity key agreement failed"); } @@ -80,7 +77,7 @@ void Peer::received( const uint64_t now = RR->node->now(); #ifdef ZT_ENABLE_CLUSTER - bool suboptimalPath = false; + bool isClusterSuboptimalPath = false; if ((RR->cluster)&&(hops == 0)) { // Note: findBetterEndpoint() is first since we still want to check // for a better endpoint even if we don't actually send a redirect. @@ -120,7 +117,7 @@ void Peer::received( outp.armor(_key,true,path->nextOutgoingCounter()); path->send(RR,tPtr,outp.data(),outp.size(),now); } - suboptimalPath = true; + isClusterSuboptimalPath = true; } } #endif @@ -146,78 +143,71 @@ void Peer::received( path->updateLinkQuality((unsigned int)(packetId & 7)); if (hops == 0) { - bool pathIsConfirmed = false; + bool pathAlreadyKnown = false; { Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address() == path->address()) { - _paths[p].lastReceive = now; - _paths[p].path = path; // local address may have changed! + if ((path->address().ss_family == AF_INET)&&(_v4Path.p)) { + const struct sockaddr_in *const r = reinterpret_cast(&(path->address())); + const struct sockaddr_in *const l = reinterpret_cast(&(_v4Path.p->address())); + const struct sockaddr_in *const rl = reinterpret_cast(&(path->localAddress())); + const struct sockaddr_in *const ll = reinterpret_cast(&(_v4Path.p->localAddress())); + if ((r->sin_addr.s_addr == l->sin_addr.s_addr)&&(r->sin_port == l->sin_port)&&(rl->sin_addr.s_addr == ll->sin_addr.s_addr)&&(rl->sin_port == ll->sin_port)) { + _v4Path.lr = now; #ifdef ZT_ENABLE_CLUSTER - _paths[p].localClusterSuboptimal = suboptimalPath; + _v4Path.localClusterSuboptimal = isClusterSuboptimalPath; #endif - pathIsConfirmed = true; - break; + pathAlreadyKnown = true; + } + } else if ((path->address().ss_family == AF_INET6)&&(_v6Path.p)) { + const struct sockaddr_in6 *const r = reinterpret_cast(&(path->address())); + const struct sockaddr_in6 *const l = reinterpret_cast(&(_v6Path.p->address())); + const struct sockaddr_in6 *const rl = reinterpret_cast(&(path->localAddress())); + const struct sockaddr_in6 *const ll = reinterpret_cast(&(_v6Path.p->localAddress())); + if ((!memcmp(r->sin6_addr.s6_addr,l->sin6_addr.s6_addr,16))&&(r->sin6_port == l->sin6_port)&&(!memcmp(rl->sin6_addr.s6_addr,ll->sin6_addr.s6_addr,16))&&(rl->sin6_port == ll->sin6_port)) { + _v6Path.lr = now; +#ifdef ZT_ENABLE_CLUSTER + _v6Path.localClusterSuboptimal = isClusterSuboptimalPath; +#endif + pathAlreadyKnown = true; } } } - if ( (!pathIsConfirmed) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { - if (verb == Packet::VERB_OK) { - Mutex::Lock _l(_paths_m); - - // Since this is a new path, figure out where to put it (possibly replacing an old/dead one) - unsigned int slot; - if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) { - slot = _numPaths++; - } else { - // First try to replace the worst within the same address family, if possible - int worstSlot = -1; - uint64_t worstScore = 0xffffffffffffffffULL; - for(unsigned int p=0;p<_numPaths;++p) { - if (_paths[p].path->address().ss_family == path->address().ss_family) { - const uint64_t s = _pathScore(p,now); - if (s < worstScore) { - worstScore = s; - worstSlot = (int)p; - } - } - } - if (worstSlot >= 0) { - slot = (unsigned int)worstSlot; - } else { - // If we can't find one with the same family, replace the worst of any family - slot = ZT_MAX_PEER_NETWORK_PATHS - 1; - for(unsigned int p=0;p<_numPaths;++p) { - const uint64_t s = _pathScore(p,now); - if (s < worstScore) { - worstScore = s; - slot = p; - } - } - } + if ( (!pathAlreadyKnown) && (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localAddress(),path->address())) ) { + Mutex::Lock _l(_paths_m); + _PeerPath *potentialNewPeerPath = (_PeerPath *)0; + if (path->address().ss_family == AF_INET) { + if ( (!_v4Path.p) || (!_v4Path.p->alive(now)) || ((_v4Path.p->address() != _v4ClusterPreferred)&&(path->preferenceRank() >= _v4Path.p->preferenceRank())) ) { + potentialNewPeerPath = &_v4Path; } - - _paths[slot].lastReceive = now; - _paths[slot].path = path; + } else if (path->address().ss_family == AF_INET6) { + if ( (!_v6Path.p) || (!_v6Path.p->alive(now)) || ((_v6Path.p->address() != _v6ClusterPreferred)&&(path->preferenceRank() >= _v6Path.p->preferenceRank())) ) { + potentialNewPeerPath = &_v6Path; + } + } + if (potentialNewPeerPath) { + if (verb == Packet::VERB_OK) { + potentialNewPeerPath->lr = now; + potentialNewPeerPath->p = path; #ifdef ZT_ENABLE_CLUSTER - _paths[slot].localClusterSuboptimal = suboptimalPath; - if (RR->cluster) - RR->cluster->broadcastHavePeer(_id); + potentialNewPeerPath->localClusterSuboptimal = isClusterSuboptimalPath; + if (RR->cluster) + RR->cluster->broadcastHavePeer(_id); #endif - } else { - TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); - attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); - path->sent(now); + } else { + TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),path->address().toString().c_str()); + attemptToContactAt(tPtr,path->localAddress(),path->address(),now,true,path->nextOutgoingCounter()); + path->sent(now); + } } } } else if (this->trustEstablished(now)) { // Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership) #ifdef ZT_ENABLE_CLUSTER - // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection - const bool haveCluster = (RR->cluster); + // Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection + const bool haveCluster = (RR->cluster); #else - const bool haveCluster = false; + const bool haveCluster = false; #endif if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) { _lastDirectPathPushSent = now; @@ -290,60 +280,50 @@ void Peer::received( } } -bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if ( (_paths[p].path->address() == addr) && ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) - return true; - } - return false; -} - -bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead) +bool Peer::sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)||(forceEvenIfDead)) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; - } + uint64_t v6lr = 0; + if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if ( (v6lr > v4lr) && ((now - v6lr) < ZT_PATH_ALIVE_TIMEOUT) ) { + return _v6Path.p->send(RR,tPtr,data,len,now); + } else if ((now - v4lr) < ZT_PATH_ALIVE_TIMEOUT) { + return _v4Path.p->send(RR,tPtr,data,len,now); + } else if (force) { + if (v6lr > v4lr) { + return _v6Path.p->send(RR,tPtr,data,len,now); + } else if (v4lr) { + return _v4Path.p->send(RR,tPtr,data,len,now); } } - if (bestp >= 0) { - return _paths[bestp].path->send(RR,tPtr,data,len,now); - } else { - return false; - } + return false; } SharedPtr Peer::getBestPath(uint64_t now,bool includeExpired) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) || (includeExpired) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; - } - } + uint64_t v6lr = 0; + if ( ( includeExpired || ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ( includeExpired || ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if (v6lr > v4lr) { + return _v6Path.p; + } else if (v4lr) { + return _v4Path.p; } - if (bestp >= 0) { - return _paths[bestp].path; - } else { - return SharedPtr(); - } + return SharedPtr(); } void Peer::sendHELLO(void *tPtr,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int counter) @@ -420,79 +400,44 @@ bool Peer::doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily) { Mutex::Lock _l(_paths_m); - int bestp = -1; - uint64_t best = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && ((inetAddressFamily < 0)||((int)_paths[p].path->address().ss_family == inetAddressFamily)) ) { - const uint64_t s = _pathScore(p,now); - if (s >= best) { - best = s; - bestp = (int)p; + if (inetAddressFamily < 0) { + uint64_t v6lr = 0; + if ( ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v6Path.p) ) + v6lr = _v6Path.p->lastIn(); + uint64_t v4lr = 0; + if ( ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) && (_v4Path.p) ) + v4lr = _v4Path.p->lastIn(); + + if (v6lr > v4lr) { + if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + return true; + } + } else if (v4lr) { + if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + return true; + } + } + } else { + if ( (inetAddressFamily == AF_INET) && ((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { + if ( ((now - _v4Path.lr) >= ZT_PEER_PING_PERIOD) || (_v4Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + return true; + } + } else if ( (inetAddressFamily == AF_INET6) && ((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION) ) { + if ( ((now - _v6Path.lr) >= ZT_PEER_PING_PERIOD) || (_v6Path.p->needsHeartbeat(now)) ) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + return true; } } } - if (bestp >= 0) { - if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) { - attemptToContactAt(tPtr,_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now,false,_paths[bestp].path->nextOutgoingCounter()); - _paths[bestp].path->sent(now); - } - return true; - } else { - return false; - } -} - -bool Peer::hasActiveDirectPath(uint64_t now) const -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if (((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION)&&(_paths[p].path->alive(now))) - return true; - } return false; } -void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) -{ - Mutex::Lock _l(_paths_m); - for(unsigned int p=0;p<_numPaths;++p) { - if ( (_paths[p].path->address().ss_family == inetAddressFamily) && (_paths[p].path->address().ipScope() == scope) ) { - attemptToContactAt(tPtr,_paths[p].path->localAddress(),_paths[p].path->address(),now,false,_paths[p].path->nextOutgoingCounter()); - _paths[p].path->sent(now); - _paths[p].lastReceive = 0; // path will not be used unless it speaks again - } - } -} - -void Peer::getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const -{ - Mutex::Lock _l(_paths_m); - - int bestp4 = -1,bestp6 = -1; - uint64_t best4 = 0ULL,best6 = 0ULL; - for(unsigned int p=0;p<_numPaths;++p) { - if ( ((now - _paths[p].lastReceive) <= ZT_PEER_PATH_EXPIRATION) && (_paths[p].path->alive(now)) ) { - if (_paths[p].path->address().ss_family == AF_INET) { - const uint64_t s = _pathScore(p,now); - if (s >= best4) { - best4 = s; - bestp4 = (int)p; - } - } else if (_paths[p].path->address().ss_family == AF_INET6) { - const uint64_t s = _pathScore(p,now); - if (s >= best6) { - best6 = s; - bestp6 = (int)p; - } - } - } - } - - if (bestp4 >= 0) - v4 = _paths[bestp4].path->address(); - if (bestp6 >= 0) - v6 = _paths[bestp6].path->address(); -} - } // namespace ZeroTier diff --git a/zto/node/Peer.hpp b/zto/node/Peer.hpp index 4183641..9b57f23 100644 --- a/zto/node/Peer.hpp +++ b/zto/node/Peer.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PEER_HPP @@ -108,20 +116,10 @@ public: * @param addr Remote address * @return True if we have an active path to this destination */ - bool hasActivePathTo(uint64_t now,const InetAddress &addr) const; - - /** - * Set which known path for an address family is optimal - * - * @param addr Address to make exclusive - */ - inline void setClusterOptimal(const InetAddress &addr) + inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const { - if (addr.ss_family == AF_INET) { - _remoteClusterOptimal4 = (uint32_t)reinterpret_cast(&addr)->sin_addr.s_addr; - } else if (addr.ss_family == AF_INET6) { - memcpy(_remoteClusterOptimal6,reinterpret_cast(&addr)->sin6_addr.s6_addr,16); - } + Mutex::Lock _l(_paths_m); + return ( ((addr.ss_family == AF_INET)&&(_v4Path.p)&&(_v4Path.p->address() == addr)&&(_v4Path.p->alive(now))) || ((addr.ss_family == AF_INET6)&&(_v6Path.p)&&(_v6Path.p->address() == addr)&&(_v6Path.p->alive(now))) ); } /** @@ -131,14 +129,17 @@ public: * @param data Packet data * @param len Packet length * @param now Current time - * @param forceEvenIfDead If true, send even if the path is not 'alive' + * @param force If true, send even if path is not alive * @return True if we actually sent something */ - bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead); + bool sendDirect(void *tPtr,const void *data,unsigned int len,uint64_t now,bool force); /** * Get the best current direct path * + * This does not check Path::alive(), but does return the most recently + * active path and does check expiration (which is a longer timeout). + * * @param now Current time * @param includeExpired If true, include even expired paths * @return Best current path or NULL if none @@ -192,12 +193,6 @@ public: */ bool doPingAndKeepalive(void *tPtr,uint64_t now,int inetAddressFamily); - /** - * @param now Current time - * @return True if this peer has at least one active and alive direct path - */ - bool hasActiveDirectPath(uint64_t now) const; - /** * Reset paths within a given IP scope and address family * @@ -209,30 +204,61 @@ public: * @param inetAddressFamily Family e.g. AF_INET * @param now Current time */ - void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now); + inline void resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,uint64_t now) + { + Mutex::Lock _l(_paths_m); + if ((inetAddressFamily == AF_INET)&&(_v4Path.lr)&&(_v4Path.p->address().ipScope() == scope)) { + attemptToContactAt(tPtr,_v4Path.p->localAddress(),_v4Path.p->address(),now,false,_v4Path.p->nextOutgoingCounter()); + _v4Path.p->sent(now); + _v4Path.lr = 0; // path will not be used unless it speaks again + } else if ((inetAddressFamily == AF_INET6)&&(_v6Path.lr)&&(_v6Path.p->address().ipScope() == scope)) { + attemptToContactAt(tPtr,_v6Path.p->localAddress(),_v6Path.p->address(),now,false,_v6Path.p->nextOutgoingCounter()); + _v6Path.p->sent(now); + _v6Path.lr = 0; // path will not be used unless it speaks again + } + } /** - * Get most recently active path addresses for IPv4 and/or IPv6 + * Indicate that the given address was provided by a cluster as a preferred destination * - * Note that v4 and v6 are not modified if they are not found, so - * initialize these to a NULL address to be able to check. + * @param addr Address cluster prefers that we use + */ + inline void setClusterPreferred(const InetAddress &addr) + { + if (addr.ss_family == AF_INET) + _v4ClusterPreferred = addr; + else if (addr.ss_family == AF_INET6) + _v6ClusterPreferred = addr; + } + + /** + * Fill parameters with V4 and V6 addresses if known and alive * * @param now Current time * @param v4 Result parameter to receive active IPv4 address, if any * @param v6 Result parameter to receive active IPv6 address, if any */ - void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const; + inline void getRendezvousAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const + { + Mutex::Lock _l(_paths_m); + if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) + v4 = _v4Path.p->address(); + if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) + v6 = _v6Path.p->address(); + } /** * @param now Current time - * @return All known direct paths to this peer and whether they are expired (true == expired) + * @return All known paths to this peer */ - inline std::vector< std::pair< SharedPtr,bool > > paths(const uint64_t now) const + inline std::vector< SharedPtr > paths(const uint64_t now) const { - std::vector< std::pair< SharedPtr,bool > > pp; + std::vector< SharedPtr > pp; Mutex::Lock _l(_paths_m); - for(unsigned int p=0,np=_numPaths;p,bool >(_paths[p].path,(now - _paths[p].lastReceive) > ZT_PEER_PATH_EXPIRATION)); + if (((now - _v4Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v4Path.p->alive(now))) + pp.push_back(_v4Path.p); + if (((now - _v6Path.lr) < ZT_PEER_PATH_EXPIRATION)&&(_v6Path.p->alive(now))) + pp.push_back(_v6Path.p); return pp; } @@ -298,11 +324,8 @@ public: */ inline bool hasLocalClusterOptimalPath(uint64_t now) const { - for(unsigned int p=0,np=_numPaths;palive(now)) && (!_paths[p].localClusterSuboptimal) ) - return true; - } - return false; + Mutex::Lock _l(_paths_m); + return ( ((_v4Path.p)&&(_v4Path.p->alive(now))&&(!_v4Path.localClusterSuboptimal)) || ((_v6Path.p)&&(_v6Path.p->alive(now))&&(!_v6Path.localClusterSuboptimal)) ); } #endif @@ -424,32 +447,19 @@ public: } private: - inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const + struct _PeerPath { - uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)); - - if (_paths[p].path->address().ss_family == AF_INET) { - s += (uint64_t)(ZT_PEER_PING_PERIOD * (unsigned long)(reinterpret_cast(&(_paths[p].path->address()))->sin_addr.s_addr == _remoteClusterOptimal4)); - } else if (_paths[p].path->address().ss_family == AF_INET6) { - uint64_t clusterWeight = ZT_PEER_PING_PERIOD; - const uint8_t *a = reinterpret_cast(reinterpret_cast(&(_paths[p].path->address()))->sin6_addr.s6_addr); - for(long i=0;i<16;++i) { - if (a[i] != _remoteClusterOptimal6[i]) { - clusterWeight = 0; - break; - } - } - s += clusterWeight; - } - - s += (ZT_PEER_PING_PERIOD / 2) * (uint64_t)_paths[p].path->alive(now); - #ifdef ZT_ENABLE_CLUSTER - s -= ZT_PEER_PING_PERIOD * (uint64_t)_paths[p].localClusterSuboptimal; + _PeerPath() : lr(0),p(),localClusterSuboptimal(false) {} +#else + _PeerPath() : lr(0),p() {} #endif - - return s; - } + uint64_t lr; // time of last valid ZeroTier packet + SharedPtr p; +#ifdef ZT_ENABLE_CLUSTER + bool localClusterSuboptimal; // true if our cluster has determined that we should not be serving this peer +#endif + }; uint8_t _key[ZT_PEER_SECRET_KEY_LENGTH]; @@ -468,26 +478,20 @@ private: uint64_t _lastCredentialsReceived; uint64_t _lastTrustEstablishedPacketReceived; - uint8_t _remoteClusterOptimal6[16]; - uint32_t _remoteClusterOptimal4; - uint16_t _vProto; uint16_t _vMajor; uint16_t _vMinor; uint16_t _vRevision; - Identity _id; + InetAddress _v4ClusterPreferred; + InetAddress _v6ClusterPreferred; - struct { - uint64_t lastReceive; - SharedPtr path; -#ifdef ZT_ENABLE_CLUSTER - bool localClusterSuboptimal; -#endif - } _paths[ZT_MAX_PEER_NETWORK_PATHS]; + _PeerPath _v4Path; // IPv4 direct path + _PeerPath _v6Path; // IPv6 direct path Mutex _paths_m; - unsigned int _numPaths; + Identity _id; + unsigned int _latency; unsigned int _directPathPushCutoffCount; unsigned int _credentialsCutoffCount; diff --git a/zto/node/Poly1305.cpp b/zto/node/Poly1305.cpp index b78071f..13d4712 100644 --- a/zto/node/Poly1305.cpp +++ b/zto/node/Poly1305.cpp @@ -135,11 +135,12 @@ typedef struct poly1305_context { unsigned char opaque[136]; } poly1305_context; -#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__)) +#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64)) ////////////////////////////////////////////////////////////////////////////// // 128-bit implementation for MSC and GCC from Poly1305-donna + #if defined(_MSC_VER) #include @@ -183,9 +184,9 @@ typedef struct poly1305_state_internal_t { unsigned char final; } poly1305_state_internal_t; -/* interpret eight 8 bit unsigned integers as a 64 bit unsigned integer in little endian */ -static inline unsigned long long -U8TO64(const unsigned char *p) { +#if defined(ZT_NO_TYPE_PUNNING) || (__BYTE_ORDER != __LITTLE_ENDIAN) +static inline unsigned long long U8TO64(const unsigned char *p) +{ return (((unsigned long long)(p[0] & 0xff) ) | ((unsigned long long)(p[1] & 0xff) << 8) | @@ -196,10 +197,13 @@ U8TO64(const unsigned char *p) { ((unsigned long long)(p[6] & 0xff) << 48) | ((unsigned long long)(p[7] & 0xff) << 56)); } +#else +#define U8TO64(p) (*reinterpret_cast(p)) +#endif -/* store a 64 bit unsigned integer as eight 8 bit unsigned integers in little endian */ -static inline void -U64TO8(unsigned char *p, unsigned long long v) { +#if defined(ZT_NO_TYPE_PUNNING) || (__BYTE_ORDER != __LITTLE_ENDIAN) +static inline void U64TO8(unsigned char *p, unsigned long long v) +{ p[0] = (v ) & 0xff; p[1] = (v >> 8) & 0xff; p[2] = (v >> 16) & 0xff; @@ -209,6 +213,9 @@ U64TO8(unsigned char *p, unsigned long long v) { p[6] = (v >> 48) & 0xff; p[7] = (v >> 56) & 0xff; } +#else +#define U64TO8(p,v) ((*reinterpret_cast(p)) = (v)) +#endif static inline void poly1305_init(poly1305_context *ctx, const unsigned char key[32]) { diff --git a/zto/node/Poly1305.hpp b/zto/node/Poly1305.hpp index 62d5754..ff70998 100644 --- a/zto/node/Poly1305.hpp +++ b/zto/node/Poly1305.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_POLY1305_HPP diff --git a/zto/node/Revocation.cpp b/zto/node/Revocation.cpp index bab5653..026058d 100644 --- a/zto/node/Revocation.cpp +++ b/zto/node/Revocation.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Revocation.hpp" diff --git a/zto/node/Revocation.hpp b/zto/node/Revocation.hpp index 8b9ce6d..e8f5d00 100644 --- a/zto/node/Revocation.hpp +++ b/zto/node/Revocation.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_REVOCATION_HPP @@ -26,6 +34,7 @@ #include "Constants.hpp" #include "../include/ZeroTierOne.h" +#include "Credential.hpp" #include "Address.hpp" #include "C25519.hpp" #include "Utils.hpp" @@ -44,20 +53,10 @@ class RuntimeEnvironment; /** * Revocation certificate to instantaneously revoke a COM, capability, or tag */ -class Revocation +class Revocation : public Credential { public: - /** - * Credential type being revoked - */ - enum CredentialType - { - CREDENTIAL_TYPE_NULL = 0, - CREDENTIAL_TYPE_COM = 1, // CertificateOfMembership - CREDENTIAL_TYPE_CAPABILITY = 2, - CREDENTIAL_TYPE_TAG = 3, - CREDENTIAL_TYPE_COO = 4 // CertificateOfOwnership - }; + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_REVOCATION; } Revocation() { @@ -73,23 +72,23 @@ public: * @param tgt Target node whose credential(s) are being revoked * @param ct Credential type being revoked */ - Revocation(const uint64_t i,const uint64_t nwid,const uint64_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const CredentialType ct) : + Revocation(const uint32_t i,const uint64_t nwid,const uint32_t cid,const uint64_t thr,const uint64_t fl,const Address &tgt,const Credential::Type ct) : _id(i), - _networkId(nwid), _credentialId(cid), + _networkId(nwid), _threshold(thr), _flags(fl), _target(tgt), _signedBy(), _type(ct) {} - inline uint64_t id() const { return _id; } + inline uint32_t id() const { return _id; } + inline uint32_t credentialId() const { return _credentialId; } inline uint64_t networkId() const { return _networkId; } - inline uint64_t credentialId() const { return _credentialId; } inline uint64_t threshold() const { return _threshold; } inline const Address &target() const { return _target; } inline const Address &signer() const { return _signedBy; } - inline CredentialType type() const { return _type; } + inline Credential::Type type() const { return _type; } inline bool fastPropagate() const { return ((_flags & ZT_REVOCATION_FLAG_FAST_PROPAGATE) != 0); } @@ -123,8 +122,10 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_id); b.append(_networkId); + b.append((uint32_t)0); // 4 unused bytes, currently set to 0 b.append(_credentialId); b.append(_threshold); b.append(_flags); @@ -151,14 +152,16 @@ public: unsigned int p = startAt; - _id = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _id = b.template at(p); p += 4; _networkId = b.template at(p); p += 8; - _credentialId = b.template at(p); p += 8; + p += 4; // 4 bytes, currently unused + _credentialId = b.template at(p); p += 4; _threshold = b.template at(p); p += 8; _flags = b.template at(p); p += 8; _target.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; _signedBy.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH; - _type = (CredentialType)b[p++]; + _type = (Credential::Type)b[p++]; if (b[p++] == 1) { if (b.template at(p) == ZT_C25519_SIGNATURE_LEN) { @@ -178,14 +181,14 @@ public: } private: - uint64_t _id; + uint32_t _id; + uint32_t _credentialId; uint64_t _networkId; - uint64_t _credentialId; uint64_t _threshold; uint64_t _flags; Address _target; Address _signedBy; - CredentialType _type; + Credential::Type _type; C25519::Signature _signature; }; diff --git a/zto/node/RuntimeEnvironment.hpp b/zto/node/RuntimeEnvironment.hpp index 7ba1c98..d8e1d69 100644 --- a/zto/node/RuntimeEnvironment.hpp +++ b/zto/node/RuntimeEnvironment.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_RUNTIMEENVIRONMENT_HPP diff --git a/zto/node/SHA512.cpp b/zto/node/SHA512.cpp index 76737d3..c8d81dd 100644 --- a/zto/node/SHA512.cpp +++ b/zto/node/SHA512.cpp @@ -1,20 +1,11 @@ +// Code taken from NaCl by D. J. Bernstein and others +// Public domain + /* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ +20080913 +D. J. Bernstein +Public domain. +*/ #include #include @@ -25,18 +16,6 @@ namespace ZeroTier { -////////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////// - -// Code taken from NaCl by D. J. Bernstein and others -// Public domain - -/* -20080913 -D. J. Bernstein -Public domain. -*/ - #define uint64 uint64_t #ifdef ZT_NO_TYPE_PUNNING diff --git a/zto/node/SHA512.hpp b/zto/node/SHA512.hpp index 639a7df..584f8e1 100644 --- a/zto/node/SHA512.hpp +++ b/zto/node/SHA512.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SHA512_HPP diff --git a/zto/node/Salsa20.cpp b/zto/node/Salsa20.cpp index 1a4641f..1d4117e 100644 --- a/zto/node/Salsa20.cpp +++ b/zto/node/Salsa20.cpp @@ -66,65 +66,49 @@ static const _s20sseconsts _S20SSECONSTANTS; namespace ZeroTier { -void Salsa20::init(const void *key,unsigned int kbits,const void *iv) - throw() +void Salsa20::init(const void *key,const void *iv) { #ifdef ZT_SALSA20_SSE - const uint32_t *k = (const uint32_t *)key; - + const uint32_t *const k = (const uint32_t *)key; _state.i[0] = 0x61707865; + _state.i[1] = 0x3320646e; + _state.i[2] = 0x79622d32; _state.i[3] = 0x6b206574; - _state.i[13] = k[0]; - _state.i[10] = k[1]; - _state.i[7] = k[2]; _state.i[4] = k[3]; - if (kbits == 256) { - k += 4; - _state.i[1] = 0x3320646e; - _state.i[2] = 0x79622d32; - } else { - _state.i[1] = 0x3120646e; - _state.i[2] = 0x79622d36; - } - _state.i[15] = k[0]; - _state.i[12] = k[1]; - _state.i[9] = k[2]; - _state.i[6] = k[3]; - _state.i[14] = ((const uint32_t *)iv)[0]; - _state.i[11] = ((const uint32_t *)iv)[1]; _state.i[5] = 0; + _state.i[6] = k[7]; + _state.i[7] = k[2]; _state.i[8] = 0; + _state.i[9] = k[6]; + _state.i[10] = k[1]; + _state.i[11] = ((const uint32_t *)iv)[1]; + _state.i[12] = k[5]; + _state.i[13] = k[0]; + _state.i[14] = ((const uint32_t *)iv)[0]; + _state.i[15] = k[4]; #else - const char *constants; - const uint8_t *k = (const uint8_t *)key; - + const char *const constants = "expand 32-byte k"; + const uint8_t *const k = (const uint8_t *)key; + _state.i[0] = U8TO32_LITTLE(constants + 0); _state.i[1] = U8TO32_LITTLE(k + 0); _state.i[2] = U8TO32_LITTLE(k + 4); _state.i[3] = U8TO32_LITTLE(k + 8); _state.i[4] = U8TO32_LITTLE(k + 12); - if (kbits == 256) { /* recommended */ - k += 16; - constants = "expand 32-byte k"; - } else { /* kbits == 128 */ - constants = "expand 16-byte k"; - } _state.i[5] = U8TO32_LITTLE(constants + 4); _state.i[6] = U8TO32_LITTLE(((const uint8_t *)iv) + 0); _state.i[7] = U8TO32_LITTLE(((const uint8_t *)iv) + 4); _state.i[8] = 0; _state.i[9] = 0; _state.i[10] = U8TO32_LITTLE(constants + 8); - _state.i[11] = U8TO32_LITTLE(k + 0); - _state.i[12] = U8TO32_LITTLE(k + 4); - _state.i[13] = U8TO32_LITTLE(k + 8); - _state.i[14] = U8TO32_LITTLE(k + 12); + _state.i[11] = U8TO32_LITTLE(k + 16); + _state.i[12] = U8TO32_LITTLE(k + 20); + _state.i[13] = U8TO32_LITTLE(k + 24); + _state.i[14] = U8TO32_LITTLE(k + 28); _state.i[15] = U8TO32_LITTLE(constants + 12); - _state.i[0] = U8TO32_LITTLE(constants + 0); #endif } void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; @@ -624,7 +608,6 @@ void Salsa20::crypt12(const void *in,void *out,unsigned int bytes) } void Salsa20::crypt20(const void *in,void *out,unsigned int bytes) - throw() { uint8_t tmp[64]; const uint8_t *m = (const uint8_t *)in; diff --git a/zto/node/Salsa20.hpp b/zto/node/Salsa20.hpp index 6405d45..bfb6d9d 100644 --- a/zto/node/Salsa20.hpp +++ b/zto/node/Salsa20.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "Constants.hpp" #include "Utils.hpp" @@ -30,30 +31,101 @@ namespace ZeroTier { class Salsa20 { public: - Salsa20() throw() {} - + Salsa20() {} ~Salsa20() { Utils::burn(&_state,sizeof(_state)); } /** - * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) + * XOR d with s + * + * This is done efficiently using e.g. SSE if available. It's used when + * alternative Salsa20 implementations are used in Packet and is here + * since this is where all the SSE stuff is already included. + * + * @param d Destination to XOR + * @param s Source bytes to XOR with destination + * @param len Length of s and d + */ + static inline void memxor(uint8_t *d,const uint8_t *s,unsigned int len) + { +#ifdef ZT_SALSA20_SSE + while (len >= 128) { + __m128i s0 = _mm_loadu_si128(reinterpret_cast(s)); + __m128i s1 = _mm_loadu_si128(reinterpret_cast(s + 16)); + __m128i s2 = _mm_loadu_si128(reinterpret_cast(s + 32)); + __m128i s3 = _mm_loadu_si128(reinterpret_cast(s + 48)); + __m128i s4 = _mm_loadu_si128(reinterpret_cast(s + 64)); + __m128i s5 = _mm_loadu_si128(reinterpret_cast(s + 80)); + __m128i s6 = _mm_loadu_si128(reinterpret_cast(s + 96)); + __m128i s7 = _mm_loadu_si128(reinterpret_cast(s + 112)); + __m128i d0 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d)); + __m128i d1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 16)); + __m128i d2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 32)); + __m128i d3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 48)); + __m128i d4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 64)); + __m128i d5 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 80)); + __m128i d6 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 96)); + __m128i d7 = _mm_loadu_si128(reinterpret_cast<__m128i *>(d + 112)); + d0 = _mm_xor_si128(d0,s0); + d1 = _mm_xor_si128(d1,s1); + d2 = _mm_xor_si128(d2,s2); + d3 = _mm_xor_si128(d3,s3); + d4 = _mm_xor_si128(d4,s4); + d5 = _mm_xor_si128(d5,s5); + d6 = _mm_xor_si128(d6,s6); + d7 = _mm_xor_si128(d7,s7); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d),d0); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 16),d1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 32),d2); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 48),d3); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 64),d4); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 80),d5); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 96),d6); + _mm_storeu_si128(reinterpret_cast<__m128i *>(d + 112),d7); + s += 128; + d += 128; + len -= 128; + } + while (len >= 16) { + _mm_storeu_si128(reinterpret_cast<__m128i *>(d),_mm_xor_si128(_mm_loadu_si128(reinterpret_cast<__m128i *>(d)),_mm_loadu_si128(reinterpret_cast(s)))); + s += 16; + d += 16; + len -= 16; + } +#else +#ifndef ZT_NO_TYPE_PUNNING + while (len >= 16) { + (*reinterpret_cast(d)) ^= (*reinterpret_cast(s)); + s += 8; + d += 8; + (*reinterpret_cast(d)) ^= (*reinterpret_cast(s)); + s += 8; + d += 8; + len -= 16; + } +#endif +#endif + while (len) { + --len; + *(d++) ^= *(s++); + } + } + + /** + * @param key 256-bit (32 byte) key * @param iv 64-bit initialization vector */ - Salsa20(const void *key,unsigned int kbits,const void *iv) - throw() + Salsa20(const void *key,const void *iv) { - init(key,kbits,iv); + init(key,iv); } /** * Initialize cipher * * @param key Key bits - * @param kbits Number of key bits: 128 or 256 (recommended) * @param iv 64-bit initialization vector */ - void init(const void *key,unsigned int kbits,const void *iv) - throw(); + void init(const void *key,const void *iv); /** * Encrypt/decrypt data using Salsa20/12 @@ -62,8 +134,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt12(const void *in,void *out,unsigned int bytes) - throw(); + void crypt12(const void *in,void *out,unsigned int bytes); /** * Encrypt/decrypt data using Salsa20/20 @@ -72,8 +143,7 @@ public: * @param out Output buffer * @param bytes Length of data */ - void crypt20(const void *in,void *out,unsigned int bytes) - throw(); + void crypt20(const void *in,void *out,unsigned int bytes); private: union { diff --git a/zto/node/SelfAwareness.cpp b/zto/node/SelfAwareness.cpp index cba84cd..c5daddc 100644 --- a/zto/node/SelfAwareness.cpp +++ b/zto/node/SelfAwareness.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/node/SelfAwareness.hpp b/zto/node/SelfAwareness.hpp index c1db0c8..63c416b 100644 --- a/zto/node/SelfAwareness.hpp +++ b/zto/node/SelfAwareness.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SELFAWARENESS_HPP diff --git a/zto/node/SharedPtr.hpp b/zto/node/SharedPtr.hpp index 1dd3b43..09010f6 100644 --- a/zto/node/SharedPtr.hpp +++ b/zto/node/SharedPtr.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SHAREDPTR_HPP diff --git a/zto/node/Switch.cpp b/zto/node/Switch.cpp index 56299a9..211b706 100644 --- a/zto/node/Switch.cpp +++ b/zto/node/Switch.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/node/Switch.hpp b/zto/node/Switch.hpp index ff35093..9793dd4 100644 --- a/zto/node/Switch.hpp +++ b/zto/node/Switch.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_N_SWITCH_HPP diff --git a/zto/node/Tag.cpp b/zto/node/Tag.cpp index 3f924da..39b17f2 100644 --- a/zto/node/Tag.cpp +++ b/zto/node/Tag.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Tag.hpp" diff --git a/zto/node/Tag.hpp b/zto/node/Tag.hpp index 3808590..746ade2 100644 --- a/zto/node/Tag.hpp +++ b/zto/node/Tag.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_TAG_HPP @@ -25,6 +33,7 @@ #include #include "Constants.hpp" +#include "Credential.hpp" #include "C25519.hpp" #include "Address.hpp" #include "Identity.hpp" @@ -51,9 +60,11 @@ class RuntimeEnvironment; * Unlike capabilities tags are signed only by the issuer and are never * transferrable. */ -class Tag +class Tag : public Credential { public: + static inline Credential::Type credentialType() { return Credential::CREDENTIAL_TYPE_TAG; } + Tag() { memset(this,0,sizeof(Tag)); @@ -67,19 +78,19 @@ public: * @param value Tag value */ Tag(const uint64_t nwid,const uint64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) : - _networkId(nwid), - _ts(ts), _id(id), _value(value), + _networkId(nwid), + _ts(ts), _issuedTo(issuedTo), _signedBy() { } - inline uint64_t networkId() const { return _networkId; } - inline uint64_t timestamp() const { return _ts; } inline uint32_t id() const { return _id; } inline const uint32_t &value() const { return _value; } + inline uint64_t networkId() const { return _networkId; } + inline uint64_t timestamp() const { return _ts; } inline const Address &issuedTo() const { return _issuedTo; } inline const Address &signedBy() const { return _signedBy; } @@ -115,11 +126,9 @@ public: { if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL); - // These are the same between Tag and Capability b.append(_networkId); b.append(_ts); b.append(_id); - b.append(_value); _issuedTo.appendTo(b); @@ -187,10 +196,10 @@ public: }; private: - uint64_t _networkId; - uint64_t _ts; uint32_t _id; uint32_t _value; + uint64_t _networkId; + uint64_t _ts; Address _issuedTo; Address _signedBy; C25519::Signature _signature; diff --git a/zto/node/Topology.cpp b/zto/node/Topology.cpp index a1d3733..80f4ed4 100644 --- a/zto/node/Topology.cpp +++ b/zto/node/Topology.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "Constants.hpp" diff --git a/zto/node/Topology.hpp b/zto/node/Topology.hpp index 4870ab5..d06ba94 100644 --- a/zto/node/Topology.hpp +++ b/zto/node/Topology.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_TOPOLOGY_HPP @@ -314,7 +322,9 @@ public: Address *a = (Address *)0; SharedPtr *p = (SharedPtr *)0; while (i.next(a,p)) { - cnt += (unsigned long)((*p)->hasActiveDirectPath(now)); + const SharedPtr pp((*p)->getBestPath(now,false)); + if ((pp)&&(pp->alive(now))) + ++cnt; } return cnt; } diff --git a/zto/node/Utils.cpp b/zto/node/Utils.cpp index fb448dd..d69e533 100644 --- a/zto/node/Utils.cpp +++ b/zto/node/Utils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -156,7 +164,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) s20Key[1] = (uint64_t)buf; // address of buf s20Key[2] = (uint64_t)s20Key; // address of s20Key[] s20Key[3] = (uint64_t)&s20; // address of s20 - s20.init(s20Key,256,s20Key); + s20.init(s20Key,s20Key); } #ifdef __WINDOWS__ @@ -177,6 +185,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } @@ -209,6 +218,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes) } randomPtr = 0; s20.crypt12(randomBuf,randomBuf,sizeof(randomBuf)); + s20.init(randomBuf,randomBuf); } ((uint8_t *)buf)[i] = randomBuf[randomPtr++]; } diff --git a/zto/node/Utils.hpp b/zto/node/Utils.hpp index ceb29d7..25a9005 100644 --- a/zto/node/Utils.hpp +++ b/zto/node/Utils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_UTILS_HPP diff --git a/zto/node/World.hpp b/zto/node/World.hpp index 6e835be..003d70e 100644 --- a/zto/node/World.hpp +++ b/zto/node/World.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_WORLD_HPP diff --git a/zto/one.cpp b/zto/one.cpp index edefe82..1f38361 100644 --- a/zto/one.cpp +++ b/zto/one.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -1257,7 +1265,7 @@ public: }; #ifdef __WINDOWS__ -int _tmain(int argc, _TCHAR* argv[]) +int __cdecl _tmain(int argc, _TCHAR* argv[]) #else int main(int argc,char **argv) #endif diff --git a/zto/osdep/Arp.cpp b/zto/osdep/Arp.cpp index fcc122f..c06f459 100644 --- a/zto/osdep/Arp.cpp +++ b/zto/osdep/Arp.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/osdep/Arp.hpp b/zto/osdep/Arp.hpp index 5f0d199..e26fcdb 100644 --- a/zto/osdep/Arp.hpp +++ b/zto/osdep/Arp.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ARP_HPP diff --git a/zto/osdep/BSDEthernetTap.cpp b/zto/osdep/BSDEthernetTap.cpp index 62fabc4..87a9aec 100644 --- a/zto/osdep/BSDEthernetTap.cpp +++ b/zto/osdep/BSDEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/osdep/BSDEthernetTap.hpp b/zto/osdep/BSDEthernetTap.hpp index 8c6314d..3cb9c10 100644 --- a/zto/osdep/BSDEthernetTap.hpp +++ b/zto/osdep/BSDEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BSDETHERNETTAP_HPP diff --git a/zto/osdep/Binder.hpp b/zto/osdep/Binder.hpp index 9829f17..ee83282 100644 --- a/zto/osdep/Binder.hpp +++ b/zto/osdep/Binder.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BINDER_HPP diff --git a/zto/osdep/BlockingQueue.hpp b/zto/osdep/BlockingQueue.hpp index 6172f4d..43ae743 100644 --- a/zto/osdep/BlockingQueue.hpp +++ b/zto/osdep/BlockingQueue.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_BLOCKINGQUEUE_HPP @@ -34,7 +42,7 @@ template class BlockingQueue { public: - BlockingQueue(void) {} + BlockingQueue(void) : r(true) {} inline void post(T t) { @@ -43,19 +51,34 @@ public: c.notify_one(); } - inline T get(void) + inline void stop(void) + { + std::lock_guard lock(m); + r = false; + c.notify_all(); + } + + /** + * @param value Value to set to next queue item if return value is true + * @return False if stop() has been called, true otherwise + */ + inline bool get(T &value) { std::unique_lock lock(m); - while(q.empty()) + if (!r) return false; + while (q.empty()) { c.wait(lock); - T val = q.front(); + if (!r) return false; + } + value = q.front(); q.pop(); - return val; + return true; } private: + volatile bool r; std::queue q; - mutable std::mutex m; + std::mutex m; std::condition_variable c; }; diff --git a/zto/osdep/Http.cpp b/zto/osdep/Http.cpp index d4f43d1..f1d3bfe 100644 --- a/zto/osdep/Http.cpp +++ b/zto/osdep/Http.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -102,14 +110,14 @@ struct HttpPhyHandler phy->close(sock); } - inline void phyOnTcpWritable(PhySocket *sock,void **uptr, bool stack_invoked) + inline void phyOnTcpWritable(PhySocket *sock,void **uptr) { - if (writePtr < writeSize) { - long n = phy->streamSend(sock,writeBuf + writePtr,writeSize - writePtr,true); + if (writePtr < (unsigned long)writeBuf.length()) { + long n = phy->streamSend(sock,writeBuf.data() + writePtr,(unsigned long)writeBuf.length() - writePtr,true); if (n > 0) writePtr += n; } - if (writePtr >= writeSize) + if (writePtr >= (unsigned long)writeBuf.length()) phy->setNotifyWritable(sock,false); } @@ -127,8 +135,7 @@ struct HttpPhyHandler unsigned long messageSize; unsigned long writePtr; uint64_t lastActivity; - unsigned long writeSize; - char writeBuf[32768]; + std::string writeBuf; unsigned long maxResponseSize; std::map *responseHeaders; @@ -236,24 +243,26 @@ unsigned int Http::_do( handler.lastActivity = OSUtils::now(); try { - handler.writeSize = Utils::snprintf(handler.writeBuf,sizeof(handler.writeBuf),"%s %s HTTP/1.1\r\n",method,path); - for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) - handler.writeSize += Utils::snprintf(handler.writeBuf + handler.writeSize,sizeof(handler.writeBuf) - handler.writeSize,"%s: %s\r\n",h->first.c_str(),h->second.c_str()); - handler.writeSize += Utils::snprintf(handler.writeBuf + handler.writeSize,sizeof(handler.writeBuf) - handler.writeSize,"\r\n"); - if ((requestBody)&&(requestBodyLength)) { - if ((handler.writeSize + requestBodyLength) > sizeof(handler.writeBuf)) { - responseBody = "request too large"; - return 0; - } - memcpy(handler.writeBuf + handler.writeSize,requestBody,requestBodyLength); - handler.writeSize += requestBodyLength; + char tmp[1024]; + Utils::snprintf(tmp,sizeof(tmp),"%s %s HTTP/1.1\r\n",method,path); + handler.writeBuf.append(tmp); + for(std::map::const_iterator h(requestHeaders.begin());h!=requestHeaders.end();++h) { + Utils::snprintf(tmp,sizeof(tmp),"%s: %s\r\n",h->first.c_str(),h->second.c_str()); + handler.writeBuf.append(tmp); } + handler.writeBuf.append("\r\n"); + if ((requestBody)&&(requestBodyLength)) + handler.writeBuf.append((const char *)requestBody,requestBodyLength); } catch ( ... ) { responseBody = "request too large"; return 0; } - handler.maxResponseSize = maxResponseSize; + if (maxResponseSize) { + handler.maxResponseSize = maxResponseSize; + } else { + handler.maxResponseSize = 2147483647; + } handler.responseHeaders = &responseHeaders; handler.responseBody = &responseBody; handler.error = false; diff --git a/zto/osdep/Http.hpp b/zto/osdep/Http.hpp index 1ecf4ee..3f98d76 100644 --- a/zto/osdep/Http.hpp +++ b/zto/osdep/Http.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_HTTP_HPP @@ -135,6 +143,39 @@ public: responseBody); } + /** + * Make HTTP PUT request + * + * It is the responsibility of the caller to set all headers. With PUT, the + * Content-Length and Content-Type headers must be set or the PUT will not + * work. + * + * @return HTTP status code or 0 on error (responseBody will contain error message) + */ + static inline unsigned int PUT( + unsigned long maxResponseSize, + unsigned long timeout, + const struct sockaddr *remoteAddress, + const char *path, + const std::map &requestHeaders, + const void *postData, + unsigned long postDataLength, + std::map &responseHeaders, + std::string &responseBody) + { + return _do( + "PUT", + maxResponseSize, + timeout, + remoteAddress, + path, + requestHeaders, + postData, + postDataLength, + responseHeaders, + responseBody); + } + private: static unsigned int _do( const char *method, diff --git a/zto/osdep/LinuxEthernetTap.cpp b/zto/osdep/LinuxEthernetTap.cpp index c4b978e..2d3891e 100644 --- a/zto/osdep/LinuxEthernetTap.cpp +++ b/zto/osdep/LinuxEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -93,19 +101,31 @@ LinuxEthernetTap::LinuxEthernetTap( memset(&ifr,0,sizeof(ifr)); // Try to recall our last device name, or pick an unused one if that fails. - bool recalledDevice = false; - std::string devmapbuf; - Dictionary<8194> devmap; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmapbuf)) { - devmap.load(devmapbuf.c_str()); - char desiredDevice[128]; - if (devmap.get(nwids,desiredDevice,sizeof(desiredDevice)) > 0) { - Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),desiredDevice); - Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); - recalledDevice = (stat(procpath,&sbuf) != 0); + std::map globalDeviceMap; + FILE *devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"r"); + if (devmapf) { + char buf[256]; + while (fgets(buf,sizeof(buf),devmapf)) { + char *x = (char *)0; + char *y = (char *)0; + char *saveptr = (char *)0; + for(char *f=Utils::stok(buf,"\r\n=",&saveptr);(f);f=Utils::stok((char *)0,"\r\n=",&saveptr)) { + if (!x) x = f; + else if (!y) y = f; + else break; + } + if ((x)&&(y)&&(x[0])&&(y[0])) + globalDeviceMap[x] = y; } + fclose(devmapf); + } + bool recalledDevice = false; + std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); + if (gdmEntry != globalDeviceMap.end()) { + Utils::scopy(ifr.ifr_name,sizeof(ifr.ifr_name),gdmEntry->second.c_str()); + Utils::snprintf(procpath,sizeof(procpath),"/proc/sys/net/ipv4/conf/%s",ifr.ifr_name); + recalledDevice = (stat(procpath,&sbuf) != 0); } - if (!recalledDevice) { int devno = 0; do { @@ -179,9 +199,16 @@ LinuxEthernetTap::LinuxEthernetTap( (void)::pipe(_shutdownSignalPipe); - devmap.erase(nwids); - devmap.add(nwids,_dev.c_str()); - OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),(const void *)devmap.data(),devmap.sizeBytes()); + globalDeviceMap[nwids] = _dev; + devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w"); + if (devmapf) { + gdmEntry = globalDeviceMap.begin(); + while (gdmEntry != globalDeviceMap.end()) { + fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str()); + ++gdmEntry; + } + fclose(devmapf); + } _thread = Thread::start(this); } diff --git a/zto/osdep/LinuxEthernetTap.hpp b/zto/osdep/LinuxEthernetTap.hpp index a2a00a7..ab9d237 100644 --- a/zto/osdep/LinuxEthernetTap.hpp +++ b/zto/osdep/LinuxEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_LINUXETHERNETTAP_HPP diff --git a/zto/osdep/ManagedRoute.cpp b/zto/osdep/ManagedRoute.cpp index 1fc6c78..fca1c29 100644 --- a/zto/osdep/ManagedRoute.cpp +++ b/zto/osdep/ManagedRoute.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "../node/Constants.hpp" @@ -57,8 +65,6 @@ #define ZT_LINUX_IP_COMMAND "/sbin/ip" #define ZT_LINUX_IP_COMMAND_2 "/usr/sbin/ip" -// NOTE: BSD is mostly tested on Apple/Mac but is likely to work on other BSD too - namespace ZeroTier { namespace { @@ -348,10 +354,42 @@ static bool _winRoute(bool del,const NET_LUID &interfaceLuid,const NET_IFINDEX & } } +static bool _winHasRoute(const NET_LUID &interfaceLuid, const NET_IFINDEX &interfaceIndex, const InetAddress &target, const InetAddress &via) +{ + MIB_IPFORWARD_ROW2 rtrow; + InitializeIpForwardEntry(&rtrow); + rtrow.InterfaceLuid.Value = interfaceLuid.Value; + rtrow.InterfaceIndex = interfaceIndex; + if (target.ss_family == AF_INET) { + rtrow.DestinationPrefix.Prefix.si_family = AF_INET; + rtrow.DestinationPrefix.Prefix.Ipv4.sin_family = AF_INET; + rtrow.DestinationPrefix.Prefix.Ipv4.sin_addr.S_un.S_addr = reinterpret_cast(&target)->sin_addr.S_un.S_addr; + if (via.ss_family == AF_INET) { + rtrow.NextHop.si_family = AF_INET; + rtrow.NextHop.Ipv4.sin_family = AF_INET; + rtrow.NextHop.Ipv4.sin_addr.S_un.S_addr = reinterpret_cast(&via)->sin_addr.S_un.S_addr; + } + } else if (target.ss_family == AF_INET6) { + rtrow.DestinationPrefix.Prefix.si_family = AF_INET6; + rtrow.DestinationPrefix.Prefix.Ipv6.sin6_family = AF_INET6; + memcpy(rtrow.DestinationPrefix.Prefix.Ipv6.sin6_addr.u.Byte, reinterpret_cast(&target)->sin6_addr.u.Byte, 16); + if (via.ss_family == AF_INET6) { + rtrow.NextHop.si_family = AF_INET6; + rtrow.NextHop.Ipv6.sin6_family = AF_INET6; + memcpy(rtrow.NextHop.Ipv6.sin6_addr.u.Byte, reinterpret_cast(&via)->sin6_addr.u.Byte, 16); + } + } else { + return false; + } + rtrow.DestinationPrefix.PrefixLength = target.netmaskBits(); + rtrow.SitePrefixLength = rtrow.DestinationPrefix.PrefixLength; + return (GetIpForwardEntry2(&rtrow) == NO_ERROR); +} + #endif // __WINDOWS__ -------------------------------------------------------- #ifndef ZT_ROUTING_SUPPORT_FOUND -#error "ManagedRoute.cpp has no support for managing routes on this platform! You'll need to check and see if one of the existing ones will work and make sure proper defines are set, or write one. Please do a Github pull request if you do this for a new OS." +#error "ManagedRoute.cpp has no support for managing routes on this platform! You'll need to check and see if one of the existing ones will work and make sure proper defines are set, or write one. Please do a GitHub pull request if you do this for a new OS." #endif } // anonymous namespace @@ -378,9 +416,10 @@ bool ManagedRoute::sync() return false; #endif - // Generate two more specific routes than target with one extra bit InetAddress leftt,rightt; - _forkTarget(_target,leftt,rightt); + if (_target.netmaskBits() == 0) // bifurcate only the default route + _forkTarget(_target,leftt,rightt); + else leftt = _target; #ifdef __BSD__ // ------------------------------------------------------------ @@ -447,26 +486,6 @@ bool ManagedRoute::sync() _routeCmd("change",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device); } - // Create a device-bound default target if there is none in the system. This - // is to allow e.g. IPv6 default route to work even if there is no native - // IPv6 on your LAN. - /* - if (_target.isDefaultRoute()) { - if (_systemVia) { - if (_applied.count(_target)) { - _applied.erase(_target); - _routeCmd("delete",_target,_via,_device,(_via) ? (const char *)0 : _device); - } - } else { - if (!_applied.count(_target)) { - _applied[_target] = true; // ifscoped - _routeCmd("add",_target,_via,_device,(_via) ? (const char *)0 : _device); - _routeCmd("change",_target,_via,_device,(_via) ? (const char *)0 : _device); - } - } - } - */ - #endif // __BSD__ ------------------------------------------------------------ #ifdef __LINUX__ // ---------------------------------------------------------- @@ -484,11 +503,11 @@ bool ManagedRoute::sync() #ifdef __WINDOWS__ // -------------------------------------------------------- - if (!_applied.count(leftt)) { + if ( (!_applied.count(leftt)) || (!_winHasRoute(interfaceLuid,interfaceIndex,leftt,_via)) ) { _applied[leftt] = false; // boolean unused _winRoute(false,interfaceLuid,interfaceIndex,leftt,_via); } - if ((rightt)&&(!_applied.count(rightt))) { + if ( (rightt) && ( (!_applied.count(rightt)) || (!_winHasRoute(interfaceLuid,interfaceIndex,rightt,_via)) ) ) { _applied[rightt] = false; // boolean unused _winRoute(false,interfaceLuid,interfaceIndex,rightt,_via); } diff --git a/zto/osdep/ManagedRoute.hpp b/zto/osdep/ManagedRoute.hpp index fd77a79..849bddf 100644 --- a/zto/osdep/ManagedRoute.hpp +++ b/zto/osdep/ManagedRoute.hpp @@ -1,3 +1,29 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + #ifndef ZT_MANAGEDROUTE_HPP #define ZT_MANAGEDROUTE_HPP diff --git a/zto/osdep/NeighborDiscovery.cpp b/zto/osdep/NeighborDiscovery.cpp index 4f63631..cd8b9b9 100644 --- a/zto/osdep/NeighborDiscovery.cpp +++ b/zto/osdep/NeighborDiscovery.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include "NeighborDiscovery.hpp" diff --git a/zto/osdep/NeighborDiscovery.hpp b/zto/osdep/NeighborDiscovery.hpp index 47831bd..2e7a68b 100644 --- a/zto/osdep/NeighborDiscovery.hpp +++ b/zto/osdep/NeighborDiscovery.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_NEIGHBORDISCOVERY_HPP diff --git a/zto/osdep/OSUtils.cpp b/zto/osdep/OSUtils.cpp index 33e143d..b7fce98 100644 --- a/zto/osdep/OSUtils.cpp +++ b/zto/osdep/OSUtils.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -73,7 +81,7 @@ bool OSUtils::redirectUnixOutputs(const char *stdoutPath,const char *stderrPath) } #endif // __UNIX_LIKE__ -std::vector OSUtils::listDirectory(const char *path) +std::vector OSUtils::listDirectory(const char *path,bool includeDirectories) { std::vector r; @@ -82,7 +90,7 @@ std::vector OSUtils::listDirectory(const char *path) WIN32_FIND_DATAA ffd; if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(),&ffd)) != INVALID_HANDLE_VALUE) { do { - if ((strcmp(ffd.cFileName,"."))&&(strcmp(ffd.cFileName,".."))&&((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)) + if ( (strcmp(ffd.cFileName,".")) && (strcmp(ffd.cFileName,"..")) && (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)||(((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0)&&(includeDirectories))) ) r.push_back(std::string(ffd.cFileName)); } while (FindNextFileA(hFind,&ffd)); FindClose(hFind); @@ -98,7 +106,7 @@ std::vector OSUtils::listDirectory(const char *path) if (readdir_r(d,&de,&dptr)) break; if (dptr) { - if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&(dptr->d_type != DT_DIR)) + if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,".."))&&((dptr->d_type != DT_DIR)||(includeDirectories))) r.push_back(std::string(dptr->d_name)); } else break; } diff --git a/zto/osdep/OSUtils.hpp b/zto/osdep/OSUtils.hpp index 2e007ef..4b9ee89 100644 --- a/zto/osdep/OSUtils.hpp +++ b/zto/osdep/OSUtils.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OSUTILS_HPP @@ -104,12 +112,11 @@ public: /** * List a directory's contents * - * This returns only files, not sub-directories. - * * @param path Path to list + * @param includeDirectories If true, include directories as well as files * @return Names of files in directory (without path prepended) */ - static std::vector listDirectory(const char *path); + static std::vector listDirectory(const char *path,bool includeDirectories = false); /** * Clean a directory of files whose last modified time is older than this diff --git a/zto/osdep/OSXEthernetTap.cpp b/zto/osdep/OSXEthernetTap.cpp index 35eac05..53c9ba9 100644 --- a/zto/osdep/OSXEthernetTap.cpp +++ b/zto/osdep/OSXEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -352,20 +360,33 @@ OSXEthernetTap::OSXEthernetTap( } // Try to reopen the last device we had, if we had one and it's still unused. + std::map globalDeviceMap; + FILE *devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"r"); + if (devmapf) { + char buf[256]; + while (fgets(buf,sizeof(buf),devmapf)) { + char *x = (char *)0; + char *y = (char *)0; + char *saveptr = (char *)0; + for(char *f=Utils::stok(buf,"\r\n=",&saveptr);(f);f=Utils::stok((char *)0,"\r\n=",&saveptr)) { + if (!x) x = f; + else if (!y) y = f; + else break; + } + if ((x)&&(y)&&(x[0])&&(y[0])) + globalDeviceMap[x] = y; + } + fclose(devmapf); + } bool recalledDevice = false; - std::string devmapbuf; - Dictionary<8194> devmap; - if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),devmapbuf)) { - devmap.load(devmapbuf.c_str()); - char desiredDevice[128]; - if (devmap.get(nwids,desiredDevice,sizeof(desiredDevice)) > 0) { - Utils::snprintf(devpath,sizeof(devpath),"/dev/%s",desiredDevice); - if (stat(devpath,&stattmp) == 0) { - _fd = ::open(devpath,O_RDWR); - if (_fd > 0) { - _dev = desiredDevice; - recalledDevice = true; - } + std::map::const_iterator gdmEntry = globalDeviceMap.find(nwids); + if (gdmEntry != globalDeviceMap.end()) { + std::string devpath("/dev/"); devpath.append(gdmEntry->second); + if (stat(devpath.c_str(),&stattmp) == 0) { + _fd = ::open(devpath.c_str(),O_RDWR); + if (_fd > 0) { + _dev = gdmEntry->second; + recalledDevice = true; } } } @@ -420,9 +441,16 @@ OSXEthernetTap::OSXEthernetTap( ++globalTapsRunning; - devmap.erase(nwids); - devmap.add(nwids,_dev.c_str()); - OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),(const void *)devmap.data(),devmap.sizeBytes()); + globalDeviceMap[nwids] = _dev; + devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w"); + if (devmapf) { + gdmEntry = globalDeviceMap.begin(); + while (gdmEntry != globalDeviceMap.end()) { + fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str()); + ++gdmEntry; + } + fclose(devmapf); + } _thread = Thread::start(this); } diff --git a/zto/osdep/OSXEthernetTap.hpp b/zto/osdep/OSXEthernetTap.hpp index 5a96c21..ed7f39c 100644 --- a/zto/osdep/OSXEthernetTap.hpp +++ b/zto/osdep/OSXEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_OSXETHERNETTAP_HPP diff --git a/zto/osdep/Phy.hpp b/zto/osdep/Phy.hpp index 5201cff..01a339e 100644 --- a/zto/osdep/Phy.hpp +++ b/zto/osdep/Phy.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_PHY_HPP @@ -917,7 +925,7 @@ public: } if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) { try { - _handler->phyOnTcpWritable((PhySocket *)&(*s),&(s->uptr), false); + _handler->phyOnTcpWritable((PhySocket *)&(*s),&(s->uptr)); } catch ( ... ) {} } } break; diff --git a/zto/osdep/PortMapper.cpp b/zto/osdep/PortMapper.cpp index d3a1938..9928617 100644 --- a/zto/osdep/PortMapper.cpp +++ b/zto/osdep/PortMapper.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_USE_MINIUPNPC diff --git a/zto/osdep/PortMapper.hpp b/zto/osdep/PortMapper.hpp index 0b8d15f..61015a0 100644 --- a/zto/osdep/PortMapper.hpp +++ b/zto/osdep/PortMapper.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_USE_MINIUPNPC diff --git a/zto/osdep/TestEthernetTap.hpp b/zto/osdep/TestEthernetTap.hpp new file mode 100644 index 0000000..afd8954 --- /dev/null +++ b/zto/osdep/TestEthernetTap.hpp @@ -0,0 +1,156 @@ +/* + * ZeroTier One - Network Virtualization Everywhere + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. + */ + +#ifndef ZT_TESTETHERNETTAP_HPP +#define ZT_TESTETHERNETTAP_HPP + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../node/Constants.hpp" +#include "../node/InetAddress.hpp" +#include "../node/MulticastGroup.hpp" +#include "../node/Mutex.hpp" +#include "../node/Utils.hpp" + +namespace ZeroTier { + +/** + * Dummy test Ethernet tap that does not actually open a device on the system + */ +class TestEthernetTap +{ +public: + TestEthernetTap( + const char *homePath, + const MAC &mac, + unsigned int mtu, + unsigned int metric, + uint64_t nwid, + const char *friendlyName, + void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int), + void *arg) : + _nwid(nwid), + _dev("zt_test_"), + _enabled(true) + { + char tmp[32]; + Utils::snprintf(tmp,sizeof(tmp),"%.16llx",(unsigned long long)_nwid); + _dev.append(tmp); +#ifdef ZT_TEST_TAP_REPORT_TO + _reportTo.fromString(ZT_TEST_TAP_REPORT_TO); + if (_reportTo.ss_family == AF_INET) + _reportsock = socket(AF_INET,SOCK_DGRAM,0); + else if (_reportTo.ss_family == AF_INET6) + _reportsock = socket(AF_INET6,SOCK_DGRAM,0); + else _reportsock = -1; +#endif + } + + ~TestEthernetTap() + { +#ifdef ZT_TEST_TAP_REPORT_TO + if (_reportsock >= 0) + close(_reportsock); +#endif + } + + inline void setEnabled(bool en) { _enabled = en; } + inline bool enabled() const { return _enabled; } + + inline bool addIp(const InetAddress &ip) + { + Mutex::Lock _l(_lock); + _ips.insert(ip); + return true; + } + + inline bool removeIp(const InetAddress &ip) + { + Mutex::Lock _l(_lock); + _ips.erase(ip); + return true; + } + + inline std::vector ips() const + { + Mutex::Lock _l(_lock); + return std::vector(_ips.begin(),_ips.end()); + } + + inline void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) + { +#ifdef ZT_TEST_TAP_REPORT_TO + char tmp[10000]; + if ((_reportsock >= 0)&&(len < (sizeof(tmp) - 22))) { + const uint64_t nwid2 = Utils::hton(_nwid); + memcpy(tmp,&nwid2,8); + from.copyTo(tmp + 8,6); + to.copyTo(tmp + 14,6); + const uint16_t etherType2 = Utils::hton((uint16_t)etherType); + memcpy(tmp + 20,ðerType2,2); + memcpy(tmp + 22,data,len); + sendto(_reportsock,tmp,len + 22,0,reinterpret_cast(&_reportTo),(_reportTo.ss_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)); + } +#endif + } + + inline std::string deviceName() const + { + return _dev; + } + + inline void setFriendlyName(const char *friendlyName) + { + } + + inline void scanMulticastGroups(std::vector &added,std::vector &removed) + { + } + +private: + uint64_t _nwid; + std::string _dev; + std::set _ips; + InetAddress _reportTo; +#ifdef ZT_TEST_TAP_REPORT_TO + int _reportsock; +#endif + bool _enabled; + Mutex _lock; +}; + +} // namespace ZeroTier + +#endif diff --git a/zto/osdep/Thread.hpp b/zto/osdep/Thread.hpp index 227c2cf..a2f0919 100644 --- a/zto/osdep/Thread.hpp +++ b/zto/osdep/Thread.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_THREAD_HPP @@ -46,7 +54,6 @@ class Thread { public: Thread() - throw() { _th = NULL; _tid = 0; @@ -54,7 +61,6 @@ public: template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; t._th = CreateThread(NULL,0,&___zt_threadMain,(LPVOID)instance,0,&t._tid); @@ -88,7 +94,7 @@ public: CancelSynchronousIo(t._th); } - inline operator bool() const throw() { return (_th != NULL); } + inline operator bool() const { return (_th != NULL); } private: HANDLE _th; @@ -123,33 +129,18 @@ class Thread { public: Thread() - throw() { - memset(&_tid,0,sizeof(_tid)); - pthread_attr_init(&_tattr); - // This corrects for systems with abnormally small defaults (musl) and also - // shrinks the stack on systems with large defaults to save a bit of memory. - pthread_attr_setstacksize(&_tattr,ZT_THREAD_MIN_STACK_SIZE); - _started = false; - } - - ~Thread() - { - pthread_attr_destroy(&_tattr); + memset(this,0,sizeof(Thread)); } Thread(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); } inline Thread &operator=(const Thread &t) - throw() { - memcpy(&_tid,&(t._tid),sizeof(_tid)); - _started = t._started; + memcpy(this,&t,sizeof(Thread)); return *this; } @@ -163,12 +154,20 @@ public: */ template static inline Thread start(C *instance) - throw(std::runtime_error) { Thread t; - t._started = true; - if (pthread_create(&t._tid,&t._tattr,&___zt_threadMain,instance)) + pthread_attr_t tattr; + pthread_attr_init(&tattr); + // This corrects for systems with abnormally small defaults (musl) and also + // shrinks the stack on systems with large defaults to save a bit of memory. + pthread_attr_setstacksize(&tattr,ZT_THREAD_MIN_STACK_SIZE); + if (pthread_create(&t._tid,&tattr,&___zt_threadMain,instance)) { + pthread_attr_destroy(&tattr); throw std::runtime_error("pthread_create() failed, unable to create thread"); + } else { + t._started = true; + pthread_attr_destroy(&tattr); + } return t; } @@ -190,11 +189,10 @@ public: */ static inline void sleep(unsigned long ms) { usleep(ms * 1000); } - inline operator bool() const throw() { return (_started); } + inline operator bool() const { return (_started); } private: pthread_t _tid; - pthread_attr_t _tattr; volatile bool _started; }; diff --git a/zto/osdep/WindowsEthernetTap.cpp b/zto/osdep/WindowsEthernetTap.cpp index 79b9d35..c37c741 100644 --- a/zto/osdep/WindowsEthernetTap.cpp +++ b/zto/osdep/WindowsEthernetTap.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/osdep/WindowsEthernetTap.hpp b/zto/osdep/WindowsEthernetTap.hpp index f2cf73f..a3c1c0c 100644 --- a/zto/osdep/WindowsEthernetTap.hpp +++ b/zto/osdep/WindowsEthernetTap.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_WINDOWSETHERNETTAP_HPP diff --git a/zto/selftest.cpp b/zto/selftest.cpp index 48625d5..209fe20 100644 --- a/zto/selftest.cpp +++ b/zto/selftest.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -25,6 +33,7 @@ #include #include #include +#include #include "node/Constants.hpp" #include "node/Hashtable.hpp" @@ -48,12 +57,18 @@ #include "osdep/OSUtils.hpp" #include "osdep/Phy.hpp" -#include "osdep/Http.hpp" #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" #include "controller/JSONDB.hpp" +#ifdef ZT_USE_X64_ASM_SALSA2012 +#include "ext/x64-salsa2012-asm/salsa2012.h" +#endif +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 +#include "ext/arm32-neon-salsa2012-asm/salsa2012.h" +#endif + #ifdef __WINDOWS__ #include #endif @@ -134,8 +149,6 @@ static const C25519TestVector C25519_TEST_VECTORS[ZT_NUM_C25519_TEST_VECTORS] = ////////////////////////////////////////////////////////////////////////////// -static unsigned char fuzzbuf[1048576]; - static int testCrypto() { unsigned char buf1[16384]; @@ -153,16 +166,16 @@ static int testCrypto() memset(buf2,0,sizeof(buf2)); memset(buf3,0,sizeof(buf3)); Salsa20 s20; - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf1,buf2,sizeof(buf1)); - s20.init("12345678123456781234567812345678",256,"12345678"); + s20.init("12345678123456781234567812345678","12345678"); s20.crypt20(buf2,buf3,sizeof(buf2)); if (memcmp(buf1,buf3,sizeof(buf1))) { std::cout << "FAIL (encrypt/decrypt test)" << std::endl; return -1; } } - Salsa20 s20(s20TV0Key,256,s20TV0Iv); + Salsa20 s20(s20TV0Key,s20TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt20(buf1,buf2,64); @@ -170,7 +183,7 @@ static int testCrypto() std::cout << "FAIL (test vector 0)" << std::endl; return -1; } - s20.init(s2012TV0Key,256,s2012TV0Iv); + s20.init(s2012TV0Key,s2012TV0Iv); memset(buf1,0,sizeof(buf1)); memset(buf2,0,sizeof(buf2)); s20.crypt12(buf1,buf2,64); @@ -191,8 +204,8 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); - double bytes = 0.0; + Salsa20 s20(s20TV0Key,s20TV0Iv); + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { s20.crypt12(bb,bb,1234567); @@ -200,17 +213,51 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; ::free((void *)bb); } +#ifdef ZT_USE_X64_ASM_SALSA2012 + std::cout << "[crypto] Benchmarking Salsa20/12 fast x64 ASM... "; std::cout.flush(); + { + unsigned char *bb = (unsigned char *)::malloc(1234567); + double bytes = 0.0; + uint64_t start = OSUtils::now(); + for(unsigned int i=0;i<200;++i) { + zt_salsa2012_amd64_xmm6(bb,1234567,s20TV0Iv,s20TV0Key); + bytes += 1234567.0; + } + uint64_t end = OSUtils::now(); + std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1024.0)) << " MiB/second" << std::endl; + ::free((void *)bb); + } +#endif + +#ifdef ZT_USE_ARM32_NEON_ASM_SALSA2012 + if (zt_arm_has_neon()) { + std::cout << "[crypto] Benchmarking Salsa20/12 fast arm32/neon ASM... "; std::cout.flush(); + { + unsigned char *bb = (unsigned char *)::malloc(1234567); + double bytes = 0.0; + uint64_t start = OSUtils::now(); + for(unsigned int i=0;i<200;++i) { + zt_salsa2012_armneon3_xor(bb,(const unsigned char *)0,1234567,s20TV0Iv,s20TV0Key); + bytes += 1234567.0; + } + uint64_t end = OSUtils::now(); + std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1024.0)) << " MiB/second" << std::endl; + ::free((void *)bb); + } + } +#endif + std::cout << "[crypto] Benchmarking Salsa20/20... "; std::cout.flush(); { unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - Salsa20 s20(s20TV0Key,256,s20TV0Iv); - double bytes = 0.0; + Salsa20 s20(s20TV0Key,s20TV0Iv); + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { s20.crypt20(bb,bb,1234567); @@ -218,7 +265,7 @@ static int testCrypto() } uint64_t end = OSUtils::now(); SHA512::hash(buf1,bb,1234567); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1024.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl; ::free((void *)bb); } @@ -248,14 +295,14 @@ static int testCrypto() unsigned char *bb = (unsigned char *)::malloc(1234567); for(unsigned int i=0;i<1234567;++i) bb[i] = (unsigned char)i; - double bytes = 0.0; + long double bytes = 0.0; uint64_t start = OSUtils::now(); for(unsigned int i=0;i<200;++i) { Poly1305::compute(buf1,bb,1234567,poly1305TV0Key); bytes += 1234567.0; } uint64_t end = OSUtils::now(); - std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl; + std::cout << ((bytes / 1048576.0) / ((long double)(end - start) / 1000.0)) << " MiB/second" << std::endl; ::free((void *)bb); } @@ -610,6 +657,7 @@ static int testOther() return -1; } +#if 0 std::cout << "[other] Testing Hashtable... "; std::cout.flush(); { Hashtable ht; @@ -773,42 +821,29 @@ static int testOther() } } std::cout << "PASS" << std::endl; - - std::cout << "[other] Testing hex encode/decode... "; std::cout.flush(); - for(unsigned int k=0;k<1000;++k) { - unsigned int flen = (rand() % 8194) + 1; - for(unsigned int i=0;i test; + Dictionary<8194> *test = new Dictionary<8194>(); char key[32][16]; char value[32][128]; + memset(key, 0, sizeof(key)); + memset(value, 0, sizeof(value)); for(unsigned int q=0;q<32;++q) { - Utils::snprintf(key[q],16,"%.8lx",(unsigned long)rand()); + Utils::snprintf(key[q],16,"%.8lx",(unsigned long)(rand() % 1000) + (q * 1000)); int r = rand() % 128; for(int x=0;xadd(key[q],value[q],r); } for(unsigned int q=0;q<1024;++q) { - //int r = rand() % 128; - int r = 31; + int r = rand() % 32; char tmp[128]; - if (test.get(key[r],tmp,sizeof(tmp)) >= 0) { + if (test->get(key[r],tmp,sizeof(tmp)) >= 0) { if (strcmp(value[r],tmp)) { - std::cout << "FAILED (invalid value)!" << std::endl; + std::cout << "FAILED (invalid value '" << value[r] << "' != '" << tmp << "')!" << std::endl; return -1; } } else { @@ -816,76 +851,30 @@ static int testOther() return -1; } } - for(unsigned int q=0;q<31;++q) { - char tmp[128]; - test.erase(key[q]); - if (test.get(key[q],tmp,sizeof(tmp)) >= 0) { - std::cout << "FAILED (key should have been erased)!" << std::endl; - return -1; - } - if (test.get(key[q+1],tmp,sizeof(tmp)) < 0) { - std::cout << "FAILED (key should NOT have been erased)!" << std::endl; - return -1; - } - } + delete test; } int foo = 0; volatile int *volatile bar = &foo; // force compiler not to optimize out test.get() below for(int k=0;k<200;++k) { int r = rand() % 8194; - unsigned char tmp[8194]; + unsigned char *tmp = new unsigned char[8194]; for(int q=0;q test((const char *)tmp); + Dictionary<8194> *test = new Dictionary<8194>((const char *)tmp); for(unsigned int q=0;q<100;++q) { char tmp[128]; for(unsigned int x=0;x<128;++x) tmp[x] = (char)(rand() & 0xff); tmp[127] = (char)0; char value[8194]; - *bar += test.get(tmp,value,sizeof(value)); + *bar += test->get(tmp,value,sizeof(value)); } + delete test; + delete[] tmp; } std::cout << "PASS (junk value to prevent optimization-out of test: " << foo << ")" << std::endl; - /* - std::cout << "[other] Testing controller/JSONDB..."; std::cout.flush(); - { - std::map db1data; - JSONDB db1("jsondb-test"); - for(unsigned int i=0;i<256;++i) { - std::string n; - for(unsigned int j=0,k=rand() % 4;j<=k;++j) { - if (j > 0) n.push_back('/'); - char foo[24]; - Utils::snprintf(foo,sizeof(foo),"%lx",rand()); - n.append(foo); - } - db1data[n] = {{"i",i}}; - db1.put(n,db1data[n]); - } - for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { - i->second["foo"] = "bar"; - db1.put(i->first,i->second); - } - JSONDB db2("jsondb-test"); - if (db1 != db2) { - std::cout << " FAILED (db1!=db2 #1)" << std::endl; - return -1; - } - for(std::map::iterator i(db1data.begin());i!=db1data.end();++i) { - db1.erase(i->first); - } - db2.reload(); - if (db1 != db2) { - std::cout << " FAILED (db1!=db2 #2)" << std::endl; - return -1; - } - } - std::cout << " PASS" << std::endl; - */ - return 0; } @@ -1037,53 +1026,8 @@ static int testPhy() return 0; } -/* -static int testHttp() -{ - std::map requestHeaders,responseHeaders; - std::string responseBody; - - InetAddress downloadZerotierDotCom; - std::vector rr(OSUtils::resolve("download.zerotier.com")); - if (rr.empty()) { - std::cout << "[http] Resolve of download.zerotier.com failed, skipping." << std::endl; - return 0; - } else { - for(std::vector::iterator r(rr.begin());r!=rr.end();++r) { - std::cout << "[http] download.zerotier.com: " << r->toString() << std::endl; - if (r->isV4()) - downloadZerotierDotCom = *r; - } - } - downloadZerotierDotCom.setPort(80); - - std::cout << "[http] GET http://download.zerotier.com/dev/1k @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - unsigned int sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/1k",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - std::cout << "[http] GET http://download.zerotier.com/dev/4m @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - requestHeaders["Host"] = "download.zerotier.com"; - sc = Http::GET(1024 * 1024 * 16,60000,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " " << responseBody.length() << " bytes "; - if (sc == 0) - std::cout << "ERROR: " << responseBody << std::endl; - else std::cout << "DONE" << std::endl; - - downloadZerotierDotCom = InetAddress("1.0.0.1/1234"); - std::cout << "[http] GET @" << downloadZerotierDotCom.toString() << " ... "; std::cout.flush(); - sc = Http::GET(1024 * 1024 * 16,2500,reinterpret_cast(&downloadZerotierDotCom),"/dev/4m",requestHeaders,responseHeaders,responseBody); - std::cout << sc << " (should be 0, time out)" << std::endl; - - return 0; -} -*/ - #ifdef __WINDOWS__ -int _tmain(int argc, _TCHAR* argv[]) +int __cdecl _tmain(int argc, _TCHAR* argv[]) #else int main(int argc,char **argv) #endif @@ -1133,6 +1077,7 @@ int main(int argc,char **argv) */ std::cout << "[info] sizeof(void *) == " << sizeof(void *) << std::endl; + std::cout << "[info] hardware concurrency == " << std::thread::hardware_concurrency() << std::endl; std::cout << "[info] sizeof(NetworkConfig) == " << sizeof(ZeroTier::NetworkConfig) << std::endl; srand((unsigned int)time(0)); @@ -1144,7 +1089,6 @@ int main(int argc,char **argv) r |= testIdentity(); r |= testCertificate(); r |= testPhy(); - //r |= testHttp(); //*/ if (r) diff --git a/zto/service/ClusterDefinition.hpp b/zto/service/ClusterDefinition.hpp index dda1a8c..9947e46 100644 --- a/zto/service/ClusterDefinition.hpp +++ b/zto/service/ClusterDefinition.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTERDEFINITION_HPP diff --git a/zto/service/ClusterGeoIpService.cpp b/zto/service/ClusterGeoIpService.cpp index 89015c5..2dcc917 100644 --- a/zto/service/ClusterGeoIpService.cpp +++ b/zto/service/ClusterGeoIpService.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifdef ZT_ENABLE_CLUSTER diff --git a/zto/service/ClusterGeoIpService.hpp b/zto/service/ClusterGeoIpService.hpp index ff2fcdb..380f944 100644 --- a/zto/service/ClusterGeoIpService.hpp +++ b/zto/service/ClusterGeoIpService.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_CLUSTERGEOIPSERVICE_HPP diff --git a/zto/service/OneService.cpp b/zto/service/OneService.cpp index f92650b..9bfb310 100644 --- a/zto/service/OneService.cpp +++ b/zto/service/OneService.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include @@ -88,16 +96,21 @@ using json = nlohmann::json; //#define ZT_BREAK_UDP #include "../controller/EmbeddedNetworkController.hpp" -#include "../node/Node.hpp" -// Include the right tap device driver for this platform -- add new platforms here +#ifdef ZT_USE_TEST_TAP + +#include "../osdep/TestEthernetTap.hpp" +namespace ZeroTier { typedef TestEthernetTap EthernetTap; } + +#else + #ifdef ZT_SDK - -// In network containers builds, use the virtual netcon endpoint instead of a tun/tap port driver -#include "../src/SocketTap.hpp" -namespace ZeroTier { typedef SocketTap EthernetTap; } - -#else // not ZT_SDK so pick a tap driver + #include "../controller/EmbeddedNetworkController.hpp" + #include "../node/Node.hpp" + // Use the virtual netcon endpoint instead of a tun/tap port driver + #include "../src/SocketTap.hpp" + namespace ZeroTier { typedef SocketTap EthernetTap; } +#else #ifdef __APPLE__ #include "../osdep/OSXEthernetTap.hpp" @@ -122,9 +135,11 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; } #endif // ZT_SERVICE_NETCON +#endif // ZT_USE_TEST_TAP + // Sanity limits for HTTP #define ZT_MAX_HTTP_MESSAGE_SIZE (1024 * 1024 * 64) -#define ZT_MAX_HTTP_CONNECTIONS 64 +#define ZT_MAX_HTTP_CONNECTIONS 65536 // Interface metric for ZeroTier taps -- this ensures that if we are on WiFi and also // bridged via ZeroTier to the same LAN traffic will (if the OS is sane) prefer WiFi. @@ -382,6 +397,7 @@ public: const std::string _homePath; std::string _authToken; + std::string _controllerDbPath; EmbeddedNetworkController *_controller; Phy _phy; Node *_node; @@ -483,6 +499,7 @@ public: OneServiceImpl(const char *hp,unsigned int port) : _homePath((hp) ? hp : ".") + ,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH) ,_controller((EmbeddedNetworkController *)0) ,_phy(this,false,true) ,_node((Node *)0) @@ -748,7 +765,7 @@ public: for(int i=0;i<3;++i) _portsBE[i] = Utils::hton((uint16_t)_ports[i]); - _controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str()); + _controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str()); _node->setNetconfMaster((void *)_controller); #ifdef ZT_ENABLE_CLUSTER @@ -1005,8 +1022,7 @@ public: for(it = _nets.begin(); it != _nets.end(); it++) { if(it->second.tap) { for(int j=0; jsecond.tap->_ips.size(); j++) { - if(it->second.tap->_ips[j].isEqualPrefix(addr) - || it->second.tap->_ips[j].ipsEqual(addr)) { + if(it->second.tap->_ips[j].isEqualPrefix(addr) || it->second.tap->_ips[j].ipsEqual(addr)) { return it->second.tap; } } @@ -1044,7 +1060,7 @@ public: std::map::const_iterator n(_nets.find(nwid)); if (n == _nets.end()) return false; - memcpy(&settings,&(n->second.settings),sizeof(NetworkSettings)); + settings = n->second.settings; return true; } @@ -1055,7 +1071,7 @@ public: std::map::iterator n(_nets.find(nwid)); if (n == _nets.end()) return false; - memcpy(&(n->second.settings),&settings,sizeof(NetworkSettings)); + n->second.settings = settings; char nlcpath[256]; Utils::snprintf(nlcpath,sizeof(nlcpath),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf",_homePath.c_str(),nwid); @@ -1199,9 +1215,11 @@ public: #else settings["portMappingEnabled"] = false; // not supported in build #endif - //settings["softwareUpdate"] = OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT); - //settings["softwareUpdateChannel"] = OSUtils::jsonString(settings["softwareUpdateChannel"],ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL); +#ifndef ZT_SDK + settings["softwareUpdate"] = OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT); + settings["softwareUpdateChannel"] = OSUtils::jsonString(settings["softwareUpdateChannel"],ZT_SOFTWARE_UPDATE_DEFAULT_CHANNEL); +#endif const World planet(_node->planet()); res["planetWorldId"] = planet.id(); res["planetWorldTimestamp"] = planet.timestamp(); @@ -1548,7 +1566,8 @@ public: _primaryPort = (unsigned int)OSUtils::jsonInt(settings["primaryPort"],(uint64_t)_primaryPort) & 0xffff; _portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true); -/* + +#ifndef ZT_SDK const std::string up(OSUtils::jsonString(settings["softwareUpdate"],ZT_SOFTWARE_UPDATE_DEFAULT)); const bool udist = OSUtils::jsonBool(settings["softwareUpdateDist"],false); if (((up == "apply")||(up == "download"))||(udist)) { @@ -1562,7 +1581,8 @@ public: _updater = (SoftwareUpdater *)0; _updateAutoApply = false; } -*/ +#endif + json &ignoreIfs = settings["interfacePrefixBlacklist"]; if (ignoreIfs.is_array()) { for(unsigned long i=0;i(*uptr); Mutex::Lock _l(tc->writeBuf_m); diff --git a/zto/service/OneService.hpp b/zto/service/OneService.hpp index ed3008a..b770a3c 100644 --- a/zto/service/OneService.hpp +++ b/zto/service/OneService.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_ONESERVICE_HPP @@ -23,14 +31,13 @@ #include #include "../node/InetAddress.hpp" -#include "../node/Node.hpp" -// Include the right tap device driver for this platform -- add new platforms here #ifdef ZT_SDK - // In network containers builds, use the virtual netcon endpoint instead of a tun/tap port driver + #include "../node/Node.hpp" + // Use the virtual netcon endpoint instead of a tun/tap port driver #include "../src/SocketTap.hpp" namespace ZeroTier { typedef SocketTap EthernetTap; } -#endif // not ZT_SDK so pick a tap driver +#endif namespace ZeroTier { diff --git a/zto/service/SoftwareUpdater.cpp b/zto/service/SoftwareUpdater.cpp index 7ec377c..d94beab 100644 --- a/zto/service/SoftwareUpdater.cpp +++ b/zto/service/SoftwareUpdater.cpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #include diff --git a/zto/service/SoftwareUpdater.hpp b/zto/service/SoftwareUpdater.hpp index 4bb0ef5..ff3e36d 100644 --- a/zto/service/SoftwareUpdater.hpp +++ b/zto/service/SoftwareUpdater.hpp @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef ZT_SOFTWAREUPDATER_HPP diff --git a/zto/tcp-proxy/Makefile b/zto/tcp-proxy/Makefile deleted file mode 100644 index af4e71e..0000000 --- a/zto/tcp-proxy/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -CXX=$(shell which clang++ g++ c++ 2>/dev/null | head -n 1) - -all: - $(CXX) -O3 -fno-rtti -o tcp-proxy tcp-proxy.cpp - -clean: - rm -f *.o tcp-proxy *.dSYM diff --git a/zto/tcp-proxy/README.md b/zto/tcp-proxy/README.md deleted file mode 100644 index 6f347d6..0000000 --- a/zto/tcp-proxy/README.md +++ /dev/null @@ -1,4 +0,0 @@ -TCP Proxy Server -====== - -This is the TCP proxy server we run for TCP tunneling from peers behind fascist NATs. Regular users won't have much use for this. diff --git a/zto/tcp-proxy/tcp-proxy.cpp b/zto/tcp-proxy/tcp-proxy.cpp deleted file mode 100644 index a7906aa..0000000 --- a/zto/tcp-proxy/tcp-proxy.cpp +++ /dev/null @@ -1,317 +0,0 @@ -/* - * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -// HACK! Will eventually use epoll() or something in Phy<> instead of select(). -// Also be sure to change ulimit -n and fs.file-max in /etc/sysctl.conf on relays. -#if defined(__linux__) || defined(__LINUX__) || defined(__LINUX) || defined(LINUX) -#include -#include -#undef __FD_SETSIZE -#define __FD_SETSIZE 1048576 -#undef FD_SETSIZE -#define FD_SETSIZE 1048576 -#endif - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "../osdep/Phy.hpp" - -#define ZT_TCP_PROXY_CONNECTION_TIMEOUT_SECONDS 300 -#define ZT_TCP_PROXY_TCP_PORT 443 - -using namespace ZeroTier; - -/* - * ZeroTier TCP Proxy Server - * - * This implements a simple packet encapsulation that is designed to look like - * a TLS connection. It's not a TLS connection, but it sends TLS format record - * headers. It could be extended in the future to implement a fake TLS - * handshake. - * - * At the moment, each packet is just made to look like TLS application data: - * <[1] TLS content type> - currently 0x17 for "application data" - * <[1] TLS major version> - currently 0x03 for TLS 1.2 - * <[1] TLS minor version> - currently 0x03 for TLS 1.2 - * <[2] payload length> - 16-bit length of payload in bytes - * <[...] payload> - Message payload - * - * TCP is inherently inefficient for encapsulating Ethernet, since TCP and TCP - * like protocols over TCP lead to double-ACKs. So this transport is only used - * to enable access when UDP or other datagram protocols are not available. - * - * Clients send a greeting, which is a four-byte message that contains: - * <[1] ZeroTier major version> - * <[1] minor version> - * <[2] revision> - * - * If a client has sent a greeting, it uses the new version of this protocol - * in which every encapsulated ZT packet is prepended by an IP address where - * it should be forwarded (or where it came from for replies). This causes - * this proxy to act as a remote UDP socket similar to a socks proxy, which - * will allow us to move this function off the rootservers and onto dedicated - * proxy nodes. - * - * Older ZT clients that do not send this message get their packets relayed - * to/from 127.0.0.1:9993, which will allow them to talk to and relay via - * the ZT node on the same machine as the proxy. We'll only support this for - * as long as such nodes appear to be in the wild. - */ - -struct TcpProxyService; -struct TcpProxyService -{ - Phy *phy; - int udpPortCounter; - struct Client - { - char tcpReadBuf[131072]; - char tcpWriteBuf[131072]; - unsigned long tcpWritePtr; - unsigned long tcpReadPtr; - PhySocket *tcp; - PhySocket *udp; - time_t lastActivity; - bool newVersion; - }; - std::map< PhySocket *,Client > clients; - - PhySocket *getUnusedUdp(void *uptr) - { - for(int i=0;i<65535;++i) { - ++udpPortCounter; - if (udpPortCounter > 0xfffe) - udpPortCounter = 1024; - struct sockaddr_in laddr; - memset(&laddr,0,sizeof(struct sockaddr_in)); - laddr.sin_family = AF_INET; - laddr.sin_port = htons((uint16_t)udpPortCounter); - PhySocket *udp = phy->udpBind(reinterpret_cast(&laddr),uptr); - if (udp) - return udp; - } - return (PhySocket *)0; - } - - void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len) - { - if (!*uptr) - return; - if ((from->sa_family == AF_INET)&&(len >= 16)&&(len < 2048)) { - Client &c = *((Client *)*uptr); - c.lastActivity = time((time_t *)0); - - unsigned long mlen = len; - if (c.newVersion) - mlen += 7; // new clients get IP info - - if ((c.tcpWritePtr + 5 + mlen) <= sizeof(c.tcpWriteBuf)) { - if (!c.tcpWritePtr) - phy->setNotifyWritable(c.tcp,true); - - c.tcpWriteBuf[c.tcpWritePtr++] = 0x17; // look like TLS data - c.tcpWriteBuf[c.tcpWritePtr++] = 0x03; // look like TLS 1.2 - c.tcpWriteBuf[c.tcpWritePtr++] = 0x03; // look like TLS 1.2 - - c.tcpWriteBuf[c.tcpWritePtr++] = (char)((mlen >> 8) & 0xff); - c.tcpWriteBuf[c.tcpWritePtr++] = (char)(mlen & 0xff); - - if (c.newVersion) { - c.tcpWriteBuf[c.tcpWritePtr++] = (char)4; // IPv4 - *((uint32_t *)(c.tcpWriteBuf + c.tcpWritePtr)) = ((const struct sockaddr_in *)from)->sin_addr.s_addr; - c.tcpWritePtr += 4; - *((uint16_t *)(c.tcpWriteBuf + c.tcpWritePtr)) = ((const struct sockaddr_in *)from)->sin_port; - c.tcpWritePtr += 2; - } - - for(unsigned long i=0;i %.16llx\n",inet_ntoa(reinterpret_cast(from)->sin_addr),(int)ntohs(reinterpret_cast(from)->sin_port),(unsigned long long)&c); - } - } - - void phyOnTcpConnect(PhySocket *sock,void **uptr,bool success) - { - // unused, we don't initiate outbound connections - } - - void phyOnTcpAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN,const struct sockaddr *from) - { - Client &c = clients[sockN]; - PhySocket *udp = getUnusedUdp((void *)&c); - if (!udp) { - phy->close(sockN); - clients.erase(sockN); - //printf("** TCP rejected, no more UDP ports to assign\n"); - return; - } - c.tcpWritePtr = 0; - c.tcpReadPtr = 0; - c.tcp = sockN; - c.udp = udp; - c.lastActivity = time((time_t *)0); - c.newVersion = false; - *uptrN = (void *)&c; - //printf("<< TCP from %s -> %.16llx\n",inet_ntoa(reinterpret_cast(from)->sin_addr),(unsigned long long)&c); - } - - void phyOnTcpClose(PhySocket *sock,void **uptr) - { - if (!*uptr) - return; - Client &c = *((Client *)*uptr); - phy->close(c.udp); - clients.erase(sock); - //printf("** TCP %.16llx closed\n",(unsigned long long)*uptr); - } - - void phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len) - { - Client &c = *((Client *)*uptr); - c.lastActivity = time((time_t *)0); - - for(unsigned long i=0;i= sizeof(c.tcpReadBuf)) { - phy->close(sock); - return; - } - c.tcpReadBuf[c.tcpReadPtr++] = ((const char *)data)[i]; - - if (c.tcpReadPtr >= 5) { - unsigned long mlen = ( ((((unsigned long)c.tcpReadBuf[3]) & 0xff) << 8) | (((unsigned long)c.tcpReadBuf[4]) & 0xff) ); - if (c.tcpReadPtr >= (mlen + 5)) { - if (mlen == 4) { - // Right now just sending this means the client is 'new enough' for the IP header - c.newVersion = true; - //printf("<< TCP %.16llx HELLO\n",(unsigned long long)*uptr); - } else if (mlen >= 7) { - char *payload = c.tcpReadBuf + 5; - unsigned long payloadLen = mlen; - - struct sockaddr_in dest; - memset(&dest,0,sizeof(dest)); - if (c.newVersion) { - if (*payload == (char)4) { - // New clients tell us where their packets go. - ++payload; - dest.sin_family = AF_INET; - dest.sin_addr.s_addr = *((uint32_t *)payload); - payload += 4; - dest.sin_port = *((uint16_t *)payload); // will be in network byte order already - payload += 2; - payloadLen -= 7; - } - } else { - // For old clients we will just proxy everything to a local ZT instance. The - // fact that this will come from 127.0.0.1 will in turn prevent that instance - // from doing unite() with us. It'll just forward. There will not be many of - // these. - dest.sin_family = AF_INET; - dest.sin_addr.s_addr = htonl(0x7f000001); // 127.0.0.1 - dest.sin_port = htons(9993); - } - - // Note: we do not relay to privileged ports... just an abuse prevention rule. - if ((ntohs(dest.sin_port) > 1024)&&(payloadLen >= 16)) { - phy->udpSend(c.udp,(const struct sockaddr *)&dest,payload,payloadLen); - //printf(">> TCP %.16llx to %s:%d\n",(unsigned long long)*uptr,inet_ntoa(dest.sin_addr),(int)ntohs(dest.sin_port)); - } - } - - memmove(c.tcpReadBuf,c.tcpReadBuf + (mlen + 5),c.tcpReadPtr -= (mlen + 5)); - } - } - } - } - - void phyOnTcpWritable(PhySocket *sock,void **uptr) - { - Client &c = *((Client *)*uptr); - if (c.tcpWritePtr) { - long n = phy->streamSend(sock,c.tcpWriteBuf,c.tcpWritePtr); - if (n > 0) { - memmove(c.tcpWriteBuf,c.tcpWriteBuf + n,c.tcpWritePtr -= (unsigned long)n); - if (!c.tcpWritePtr) - phy->setNotifyWritable(sock,false); - } - } else phy->setNotifyWritable(sock,false); - } - - void doHousekeeping() - { - std::vector toClose; - time_t now = time((time_t *)0); - for(std::map< PhySocket *,Client >::iterator c(clients.begin());c!=clients.end();++c) { - if ((now - c->second.lastActivity) >= ZT_TCP_PROXY_CONNECTION_TIMEOUT_SECONDS) { - toClose.push_back(c->first); - toClose.push_back(c->second.udp); - } - } - for(std::vector::iterator s(toClose.begin());s!=toClose.end();++s) - phy->close(*s); - } -}; - -int main(int argc,char **argv) -{ - signal(SIGPIPE,SIG_IGN); - signal(SIGHUP,SIG_IGN); - srand(time((time_t *)0)); - - TcpProxyService svc; - Phy phy(&svc,false,true); - svc.phy = &phy; - svc.udpPortCounter = 1023; - - { - struct sockaddr_in laddr; - memset(&laddr,0,sizeof(laddr)); - laddr.sin_family = AF_INET; - laddr.sin_port = htons(ZT_TCP_PROXY_TCP_PORT); - if (!phy.tcpListen((const struct sockaddr *)&laddr)) { - fprintf(stderr,"%s: fatal error: unable to bind TCP port %d\n",argv[0],ZT_TCP_PROXY_TCP_PORT); - return 1; - } - } - - time_t lastDidHousekeeping = time((time_t *)0); - for(;;) { - phy.poll(120000); - time_t now = time((time_t *)0); - if ((now - lastDidHousekeeping) > 120) { - lastDidHousekeeping = now; - svc.doHousekeeping(); - } - } - - return 0; -} diff --git a/zto/version.h b/zto/version.h index 3d71f6d..b3b2fc8 100644 --- a/zto/version.h +++ b/zto/version.h @@ -1,6 +1,6 @@ /* * ZeroTier One - Network Virtualization Everywhere - * Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/ + * Copyright (C) 2011-2017 ZeroTier, Inc. https://www.zerotier.com/ * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,6 +14,14 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . + * + * -- + * + * You can be released from the requirements of the license by purchasing + * a commercial license. Buying such a license is mandatory as soon as you + * develop commercial closed-source software that incorporates or links + * directly against ZeroTier software without disclosing the source code + * of your own application. */ #ifndef _ZT_VERSION_H @@ -32,7 +40,7 @@ /** * Revision */ -#define ZEROTIER_ONE_VERSION_REVISION 3 +#define ZEROTIER_ONE_VERSION_REVISION 5 /** * Build version diff --git a/zto/zerotier-one.spec b/zto/zerotier-one.spec index 1003411..306a5bf 100644 --- a/zto/zerotier-one.spec +++ b/zto/zerotier-one.spec @@ -1,5 +1,5 @@ Name: zerotier-one -Version: 1.2.2 +Version: 1.2.4 Release: 1%{?dist} Summary: ZeroTier One network virtualization service @@ -145,6 +145,9 @@ esac %endif %changelog +* Mon Apr 24 2017 Adam Ierymenko - 1.2.2-0.1 +- see https://github.com/zerotier/ZeroTierOne for release notes + * Fri Mar 17 2017 Adam Ierymenko - 1.2.2-0.1 - see https://github.com/zerotier/ZeroTierOne for release notes