Peer.cpp 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882
  1. /*
  2. * Copyright (c)2019 ZeroTier, Inc.
  3. *
  4. * Use of this software is governed by the Business Source License included
  5. * in the LICENSE.TXT file in the project's root directory.
  6. *
  7. * Change Date: 2023-01-01
  8. *
  9. * On the date above, in accordance with the Business Source License, use
  10. * of this software will be governed by version 2.0 of the Apache License.
  11. */
  12. /****/
  13. #include "../version.h"
  14. #include "Constants.hpp"
  15. #include "Peer.hpp"
  16. #include "Node.hpp"
  17. #include "Switch.hpp"
  18. #include "Network.hpp"
  19. #include "SelfAwareness.hpp"
  20. #include "Packet.hpp"
  21. #include "Trace.hpp"
  22. #include "InetAddress.hpp"
  23. #include "RingBuffer.hpp"
  24. #include "Utils.hpp"
  25. namespace ZeroTier {
  26. static unsigned char s_freeRandomByteCounter = 0;
  27. Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
  28. RR(renv),
  29. _lastReceive(0),
  30. _lastNontrivialReceive(0),
  31. _lastTriedMemorizedPath(0),
  32. _lastDirectPathPushSent(0),
  33. _lastDirectPathPushReceive(0),
  34. _lastCredentialRequestSent(0),
  35. _lastWhoisRequestReceived(0),
  36. _lastEchoRequestReceived(0),
  37. _lastCredentialsReceived(0),
  38. _lastTrustEstablishedPacketReceived(0),
  39. _lastSentFullHello(0),
  40. _lastACKWindowReset(0),
  41. _lastQoSWindowReset(0),
  42. _lastMultipathCompatibilityCheck(0),
  43. _freeRandomByte((unsigned char)((uintptr_t)this >> 4) ^ ++s_freeRandomByteCounter),
  44. _uniqueAlivePathCount(0),
  45. _localMultipathSupported(false),
  46. _remoteMultipathSupported(false),
  47. _canUseMultipath(false),
  48. _vProto(0),
  49. _vMajor(0),
  50. _vMinor(0),
  51. _vRevision(0),
  52. _id(peerIdentity),
  53. _directPathPushCutoffCount(0),
  54. _credentialsCutoffCount(0),
  55. _linkIsBalanced(false),
  56. _linkIsRedundant(false),
  57. _remotePeerMultipathEnabled(false),
  58. _lastAggregateStatsReport(0),
  59. _lastAggregateAllocation(0)
  60. {
  61. if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
  62. throw ZT_EXCEPTION_INVALID_ARGUMENT;
  63. }
  64. void Peer::received(
  65. void *tPtr,
  66. const SharedPtr<Path> &path,
  67. const unsigned int hops,
  68. const uint64_t packetId,
  69. const unsigned int payloadLength,
  70. const Packet::Verb verb,
  71. const uint64_t inRePacketId,
  72. const Packet::Verb inReVerb,
  73. const bool trustEstablished,
  74. const uint64_t networkId)
  75. {
  76. const int64_t now = RR->node->now();
  77. _lastReceive = now;
  78. switch (verb) {
  79. case Packet::VERB_FRAME:
  80. case Packet::VERB_EXT_FRAME:
  81. case Packet::VERB_NETWORK_CONFIG_REQUEST:
  82. case Packet::VERB_NETWORK_CONFIG:
  83. case Packet::VERB_MULTICAST_FRAME:
  84. _lastNontrivialReceive = now;
  85. break;
  86. default:
  87. break;
  88. }
  89. if (trustEstablished) {
  90. _lastTrustEstablishedPacketReceived = now;
  91. path->trustedPacketReceived(now);
  92. }
  93. {
  94. Mutex::Lock _l(_paths_m);
  95. recordIncomingPacket(tPtr, path, packetId, payloadLength, verb, now);
  96. if (_canUseMultipath) {
  97. if (path->needsToSendQoS(now)) {
  98. sendQOS_MEASUREMENT(tPtr, path, path->localSocket(), path->address(), now);
  99. }
  100. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  101. if (_paths[i].p) {
  102. _paths[i].p->processBackgroundPathMeasurements(now);
  103. }
  104. }
  105. }
  106. }
  107. if (hops == 0) {
  108. // If this is a direct packet (no hops), update existing paths or learn new ones
  109. bool havePath = false;
  110. {
  111. Mutex::Lock _l(_paths_m);
  112. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  113. if (_paths[i].p) {
  114. if (_paths[i].p == path) {
  115. _paths[i].lr = now;
  116. havePath = true;
  117. break;
  118. }
  119. } else break;
  120. }
  121. }
  122. bool attemptToContact = false;
  123. if ((!havePath)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id.address(),path->localSocket(),path->address()))) {
  124. Mutex::Lock _l(_paths_m);
  125. // Paths are redundant if they duplicate an alive path to the same IP or
  126. // with the same local socket and address family.
  127. bool redundant = false;
  128. unsigned int replacePath = ZT_MAX_PEER_NETWORK_PATHS;
  129. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  130. if (_paths[i].p) {
  131. if ( (_paths[i].p->alive(now)) && ( ((_paths[i].p->localSocket() == path->localSocket())&&(_paths[i].p->address().ss_family == path->address().ss_family)) || (_paths[i].p->address().ipsEqual2(path->address())) ) ) {
  132. redundant = true;
  133. break;
  134. }
  135. // If the path is the same address and port, simply assume this is a replacement
  136. if ( (_paths[i].p->address().ipsEqual2(path->address()))) {
  137. replacePath = i;
  138. break;
  139. }
  140. } else break;
  141. }
  142. // If the path isn't a duplicate of the same localSocket AND we haven't already determined a replacePath,
  143. // then find the worst path and replace it.
  144. if (!redundant && replacePath == ZT_MAX_PEER_NETWORK_PATHS) {
  145. int replacePathQuality = 0;
  146. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  147. if (_paths[i].p) {
  148. const int q = _paths[i].p->quality(now);
  149. if (q > replacePathQuality) {
  150. replacePathQuality = q;
  151. replacePath = i;
  152. }
  153. } else {
  154. replacePath = i;
  155. break;
  156. }
  157. }
  158. }
  159. if (replacePath != ZT_MAX_PEER_NETWORK_PATHS) {
  160. if (verb == Packet::VERB_OK) {
  161. RR->t->peerLearnedNewPath(tPtr,networkId,*this,path,packetId);
  162. _paths[replacePath].lr = now;
  163. _paths[replacePath].p = path;
  164. _paths[replacePath].priority = 1;
  165. } else {
  166. attemptToContact = true;
  167. }
  168. }
  169. }
  170. if (attemptToContact) {
  171. attemptToContactAt(tPtr,path->localSocket(),path->address(),now,true);
  172. path->sent(now);
  173. RR->t->peerConfirmingUnknownPath(tPtr,networkId,*this,path,packetId,verb);
  174. }
  175. }
  176. // If we have a trust relationship periodically push a message enumerating
  177. // all known external addresses for ourselves. If we already have a path this
  178. // is done less frequently.
  179. if (this->trustEstablished(now)) {
  180. const int64_t sinceLastPush = now - _lastDirectPathPushSent;
  181. if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
  182. _lastDirectPathPushSent = now;
  183. std::vector<InetAddress> pathsToPush(RR->node->directPaths());
  184. if (!pathsToPush.empty()) {
  185. std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
  186. while (p != pathsToPush.end()) {
  187. Packet *const outp = new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
  188. outp->addSize(2); // leave room for count
  189. unsigned int count = 0;
  190. while ((p != pathsToPush.end())&&((outp->size() + 24) < 1200)) {
  191. uint8_t addressType = 4;
  192. switch(p->ss_family) {
  193. case AF_INET:
  194. break;
  195. case AF_INET6:
  196. addressType = 6;
  197. break;
  198. default: // we currently only push IP addresses
  199. ++p;
  200. continue;
  201. }
  202. outp->append((uint8_t)0); // no flags
  203. outp->append((uint16_t)0); // no extensions
  204. outp->append(addressType);
  205. outp->append((uint8_t)((addressType == 4) ? 6 : 18));
  206. outp->append(p->rawIpData(),((addressType == 4) ? 4 : 16));
  207. outp->append((uint16_t)p->port());
  208. ++count;
  209. ++p;
  210. }
  211. if (count) {
  212. outp->setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
  213. outp->compress();
  214. outp->armor(_key,true);
  215. path->send(RR,tPtr,outp->data(),outp->size(),now);
  216. }
  217. delete outp;
  218. }
  219. }
  220. }
  221. }
  222. }
  223. void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
  224. uint16_t payloadLength, const Packet::Verb verb, int64_t now)
  225. {
  226. _freeRandomByte += (unsigned char)(packetId >> 8); // grab entropy to use in path selection logic for multipath
  227. if (_canUseMultipath) {
  228. path->recordOutgoingPacket(now, packetId, payloadLength, verb);
  229. }
  230. }
  231. void Peer::recordIncomingPacket(void *tPtr, const SharedPtr<Path> &path, const uint64_t packetId,
  232. uint16_t payloadLength, const Packet::Verb verb, int64_t now)
  233. {
  234. if (_canUseMultipath) {
  235. if (path->needsToSendAck(now)) {
  236. sendACK(tPtr, path, path->localSocket(), path->address(), now);
  237. }
  238. path->recordIncomingPacket(now, packetId, payloadLength, verb);
  239. }
  240. }
  241. void Peer::computeAggregateProportionalAllocation(int64_t now)
  242. {
  243. float maxStability = 0;
  244. float totalRelativeQuality = 0;
  245. float maxThroughput = 1;
  246. float maxScope = 0;
  247. float relStability[ZT_MAX_PEER_NETWORK_PATHS];
  248. float relThroughput[ZT_MAX_PEER_NETWORK_PATHS];
  249. memset(&relStability, 0, sizeof(relStability));
  250. memset(&relThroughput, 0, sizeof(relThroughput));
  251. // Survey all paths
  252. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  253. if (_paths[i].p) {
  254. relStability[i] = _paths[i].p->lastComputedStability();
  255. relThroughput[i] = (float)_paths[i].p->maxLifetimeThroughput();
  256. maxStability = relStability[i] > maxStability ? relStability[i] : maxStability;
  257. maxThroughput = relThroughput[i] > maxThroughput ? relThroughput[i] : maxThroughput;
  258. maxScope = _paths[i].p->ipScope() > maxScope ? _paths[i].p->ipScope() : maxScope;
  259. }
  260. }
  261. // Convert to relative values
  262. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  263. if (_paths[i].p) {
  264. relStability[i] /= maxStability ? maxStability : 1;
  265. relThroughput[i] /= maxThroughput ? maxThroughput : 1;
  266. float normalized_ma = Utils::normalize((float)_paths[i].p->ackAge(now), 0, ZT_PATH_MAX_AGE, 0, 10);
  267. float age_contrib = exp((-1)*normalized_ma);
  268. float relScope = ((float)(_paths[i].p->ipScope()+1) / (maxScope + 1));
  269. float relQuality =
  270. (relStability[i] * (float)ZT_PATH_CONTRIB_STABILITY)
  271. + (fmaxf(1.0f, relThroughput[i]) * (float)ZT_PATH_CONTRIB_THROUGHPUT)
  272. + relScope * (float)ZT_PATH_CONTRIB_SCOPE;
  273. relQuality *= age_contrib;
  274. // Arbitrary cutoffs
  275. relQuality = relQuality > (1.00f / 100.0f) ? relQuality : 0.0f;
  276. relQuality = relQuality < (99.0f / 100.0f) ? relQuality : 1.0f;
  277. totalRelativeQuality += relQuality;
  278. _paths[i].p->updateRelativeQuality(relQuality);
  279. }
  280. }
  281. // Convert set of relative performances into an allocation set
  282. for(uint16_t i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  283. if (_paths[i].p) {
  284. _paths[i].p->updateComponentAllocationOfAggregateLink((unsigned char)((_paths[i].p->relativeQuality() / totalRelativeQuality) * 255));
  285. }
  286. }
  287. }
  288. int Peer::computeAggregateLinkPacketDelayVariance()
  289. {
  290. float pdv = 0.0;
  291. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  292. if (_paths[i].p) {
  293. pdv += _paths[i].p->relativeQuality() * _paths[i].p->packetDelayVariance();
  294. }
  295. }
  296. return (int)pdv;
  297. }
  298. int Peer::computeAggregateLinkMeanLatency()
  299. {
  300. int ml = 0;
  301. int pathCount = 0;
  302. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  303. if (_paths[i].p) {
  304. pathCount++;
  305. ml += (int)(_paths[i].p->relativeQuality() * _paths[i].p->meanLatency());
  306. }
  307. }
  308. return ml / pathCount;
  309. }
  310. int Peer::aggregateLinkPhysicalPathCount()
  311. {
  312. std::map<std::string, bool> ifnamemap;
  313. int pathCount = 0;
  314. int64_t now = RR->node->now();
  315. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  316. if (_paths[i].p && _paths[i].p->alive(now)) {
  317. if (!ifnamemap[_paths[i].p->getName()]) {
  318. ifnamemap[_paths[i].p->getName()] = true;
  319. pathCount++;
  320. }
  321. }
  322. }
  323. return pathCount;
  324. }
  325. int Peer::aggregateLinkLogicalPathCount()
  326. {
  327. int pathCount = 0;
  328. int64_t now = RR->node->now();
  329. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  330. if (_paths[i].p && _paths[i].p->alive(now)) {
  331. pathCount++;
  332. }
  333. }
  334. return pathCount;
  335. }
  336. SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired)
  337. {
  338. Mutex::Lock _l(_paths_m);
  339. unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS;
  340. /**
  341. * Send traffic across the highest quality path only. This algorithm will still
  342. * use the old path quality metric from protocol version 9.
  343. */
  344. if (!_canUseMultipath) {
  345. long bestPathQuality = 2147483647;
  346. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  347. if (_paths[i].p) {
  348. if ((includeExpired)||((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION)) {
  349. const long q = _paths[i].p->quality(now) / _paths[i].priority;
  350. if (q <= bestPathQuality) {
  351. bestPathQuality = q;
  352. bestPath = i;
  353. }
  354. }
  355. } else break;
  356. }
  357. if (bestPath != ZT_MAX_PEER_NETWORK_PATHS) {
  358. return _paths[bestPath].p;
  359. }
  360. return SharedPtr<Path>();
  361. }
  362. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  363. if (_paths[i].p) {
  364. _paths[i].p->processBackgroundPathMeasurements(now);
  365. }
  366. }
  367. /**
  368. * Randomly distribute traffic across all paths
  369. */
  370. int numAlivePaths = 0;
  371. int numStalePaths = 0;
  372. if (RR->node->getMultipathMode() == ZT_MULTIPATH_RANDOM) {
  373. int alivePaths[ZT_MAX_PEER_NETWORK_PATHS];
  374. int stalePaths[ZT_MAX_PEER_NETWORK_PATHS];
  375. memset(&alivePaths, -1, sizeof(alivePaths));
  376. memset(&stalePaths, -1, sizeof(stalePaths));
  377. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  378. if (_paths[i].p) {
  379. if (_paths[i].p->alive(now)) {
  380. alivePaths[numAlivePaths] = i;
  381. numAlivePaths++;
  382. }
  383. else {
  384. stalePaths[numStalePaths] = i;
  385. numStalePaths++;
  386. }
  387. }
  388. }
  389. unsigned int r = _freeRandomByte;
  390. if (numAlivePaths > 0) {
  391. int rf = r % numAlivePaths;
  392. return _paths[alivePaths[rf]].p;
  393. }
  394. else if(numStalePaths > 0) {
  395. // Resort to trying any non-expired path
  396. int rf = r % numStalePaths;
  397. return _paths[stalePaths[rf]].p;
  398. }
  399. }
  400. /**
  401. * Proportionally allocate traffic according to dynamic path quality measurements
  402. */
  403. if (RR->node->getMultipathMode() == ZT_MULTIPATH_PROPORTIONALLY_BALANCED) {
  404. if ((now - _lastAggregateAllocation) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
  405. _lastAggregateAllocation = now;
  406. computeAggregateProportionalAllocation(now);
  407. }
  408. // Randomly choose path according to their allocations
  409. float rf = _freeRandomByte;
  410. for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  411. if (_paths[i].p) {
  412. if (rf < _paths[i].p->allocation()) {
  413. bestPath = i;
  414. _pathChoiceHist.push(bestPath); // Record which path we chose
  415. break;
  416. }
  417. rf -= _paths[i].p->allocation();
  418. }
  419. }
  420. if (bestPath < ZT_MAX_PEER_NETWORK_PATHS) {
  421. return _paths[bestPath].p;
  422. }
  423. }
  424. return SharedPtr<Path>();
  425. }
  426. char *Peer::interfaceListStr()
  427. {
  428. std::map<std::string, int> ifnamemap;
  429. char tmp[32];
  430. const int64_t now = RR->node->now();
  431. char *ptr = _interfaceListStr;
  432. bool imbalanced = false;
  433. memset(_interfaceListStr, 0, sizeof(_interfaceListStr));
  434. int alivePathCount = aggregateLinkLogicalPathCount();
  435. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  436. if (_paths[i].p && _paths[i].p->alive(now)) {
  437. int ipv = _paths[i].p->address().isV4();
  438. // If this is acting as an aggregate link, check allocations
  439. float targetAllocation = 1.0f / (float)alivePathCount;
  440. float currentAllocation = 1.0f;
  441. if (alivePathCount > 1) {
  442. currentAllocation = (float)_pathChoiceHist.countValue(i) / (float)_pathChoiceHist.count();
  443. if (fabs(targetAllocation - currentAllocation) > ZT_PATH_IMBALANCE_THRESHOLD) {
  444. imbalanced = true;
  445. }
  446. }
  447. char *ipvStr = ipv ? (char*)"ipv4" : (char*)"ipv6";
  448. sprintf(tmp, "(%s, %s, %.3f)", _paths[i].p->getName(), ipvStr, currentAllocation);
  449. // Prevent duplicates
  450. if(ifnamemap[_paths[i].p->getName()] != ipv) {
  451. memcpy(ptr, tmp, strlen(tmp));
  452. ptr += strlen(tmp);
  453. *ptr = ' ';
  454. ptr++;
  455. ifnamemap[_paths[i].p->getName()] = ipv;
  456. }
  457. }
  458. }
  459. ptr--; // Overwrite trailing space
  460. if (imbalanced) {
  461. sprintf(tmp, ", is asymmetrical");
  462. memcpy(ptr, tmp, sizeof(tmp));
  463. } else {
  464. *ptr = '\0';
  465. }
  466. return _interfaceListStr;
  467. }
  468. void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &other) const
  469. {
  470. unsigned int myBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  471. unsigned int myBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  472. long myBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  473. long myBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  474. unsigned int theirBestV4ByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  475. unsigned int theirBestV6ByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  476. long theirBestV4QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  477. long theirBestV6QualityByScope[ZT_INETADDRESS_MAX_SCOPE+1];
  478. for(int i=0;i<=ZT_INETADDRESS_MAX_SCOPE;++i) {
  479. myBestV4ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS;
  480. myBestV6ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS;
  481. myBestV4QualityByScope[i] = 2147483647;
  482. myBestV6QualityByScope[i] = 2147483647;
  483. theirBestV4ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS;
  484. theirBestV6ByScope[i] = ZT_MAX_PEER_NETWORK_PATHS;
  485. theirBestV4QualityByScope[i] = 2147483647;
  486. theirBestV6QualityByScope[i] = 2147483647;
  487. }
  488. Mutex::Lock _l1(_paths_m);
  489. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  490. if (_paths[i].p) {
  491. const long q = _paths[i].p->quality(now) / _paths[i].priority;
  492. const unsigned int s = (unsigned int)_paths[i].p->ipScope();
  493. switch(_paths[i].p->address().ss_family) {
  494. case AF_INET:
  495. if (q <= myBestV4QualityByScope[s]) {
  496. myBestV4QualityByScope[s] = q;
  497. myBestV4ByScope[s] = i;
  498. }
  499. break;
  500. case AF_INET6:
  501. if (q <= myBestV6QualityByScope[s]) {
  502. myBestV6QualityByScope[s] = q;
  503. myBestV6ByScope[s] = i;
  504. }
  505. break;
  506. }
  507. } else break;
  508. }
  509. Mutex::Lock _l2(other->_paths_m);
  510. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  511. if (other->_paths[i].p) {
  512. const long q = other->_paths[i].p->quality(now) / other->_paths[i].priority;
  513. const unsigned int s = (unsigned int)other->_paths[i].p->ipScope();
  514. switch(other->_paths[i].p->address().ss_family) {
  515. case AF_INET:
  516. if (q <= theirBestV4QualityByScope[s]) {
  517. theirBestV4QualityByScope[s] = q;
  518. theirBestV4ByScope[s] = i;
  519. }
  520. break;
  521. case AF_INET6:
  522. if (q <= theirBestV6QualityByScope[s]) {
  523. theirBestV6QualityByScope[s] = q;
  524. theirBestV6ByScope[s] = i;
  525. }
  526. break;
  527. }
  528. } else break;
  529. }
  530. unsigned int mine = ZT_MAX_PEER_NETWORK_PATHS;
  531. unsigned int theirs = ZT_MAX_PEER_NETWORK_PATHS;
  532. for(int s=ZT_INETADDRESS_MAX_SCOPE;s>=0;--s) {
  533. if ((myBestV6ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)&&(theirBestV6ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)) {
  534. mine = myBestV6ByScope[s];
  535. theirs = theirBestV6ByScope[s];
  536. break;
  537. }
  538. if ((myBestV4ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)&&(theirBestV4ByScope[s] != ZT_MAX_PEER_NETWORK_PATHS)) {
  539. mine = myBestV4ByScope[s];
  540. theirs = theirBestV4ByScope[s];
  541. break;
  542. }
  543. }
  544. if (mine != ZT_MAX_PEER_NETWORK_PATHS) {
  545. unsigned int alt = (unsigned int)RR->node->prng() & 1; // randomize which hint we send first for black magickal NAT-t reasons
  546. const unsigned int completed = alt + 2;
  547. while (alt != completed) {
  548. if ((alt & 1) == 0) {
  549. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
  550. outp.append((uint8_t)0);
  551. other->_id.address().appendTo(outp);
  552. outp.append((uint16_t)other->_paths[theirs].p->address().port());
  553. if (other->_paths[theirs].p->address().ss_family == AF_INET6) {
  554. outp.append((uint8_t)16);
  555. outp.append(other->_paths[theirs].p->address().rawIpData(),16);
  556. } else {
  557. outp.append((uint8_t)4);
  558. outp.append(other->_paths[theirs].p->address().rawIpData(),4);
  559. }
  560. outp.armor(_key,true);
  561. _paths[mine].p->send(RR,tPtr,outp.data(),outp.size(),now);
  562. } else {
  563. Packet outp(other->_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
  564. outp.append((uint8_t)0);
  565. _id.address().appendTo(outp);
  566. outp.append((uint16_t)_paths[mine].p->address().port());
  567. if (_paths[mine].p->address().ss_family == AF_INET6) {
  568. outp.append((uint8_t)16);
  569. outp.append(_paths[mine].p->address().rawIpData(),16);
  570. } else {
  571. outp.append((uint8_t)4);
  572. outp.append(_paths[mine].p->address().rawIpData(),4);
  573. }
  574. outp.armor(other->_key,true);
  575. other->_paths[theirs].p->send(RR,tPtr,outp.data(),outp.size(),now);
  576. }
  577. ++alt;
  578. }
  579. }
  580. }
  581. inline void Peer::processBackgroundPeerTasks(const int64_t now)
  582. {
  583. // Determine current multipath compatibility with other peer
  584. if ((now - _lastMultipathCompatibilityCheck) >= ZT_PATH_QUALITY_COMPUTE_INTERVAL) {
  585. //
  586. // Cache number of available paths so that we can short-circuit multipath logic elsewhere
  587. //
  588. // We also take notice of duplicate paths (same IP only) because we may have
  589. // recently received a direct path push from a peer and our list might contain
  590. // a dead path which hasn't been fully recognized as such. In this case we
  591. // don't want the duplicate to trigger execution of multipath code prematurely.
  592. //
  593. // This is done to support the behavior of auto multipath enable/disable
  594. // without user intervention.
  595. //
  596. int currAlivePathCount = 0;
  597. int duplicatePathsFound = 0;
  598. for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  599. if (_paths[i].p) {
  600. currAlivePathCount++;
  601. for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
  602. if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) {
  603. duplicatePathsFound+=1;
  604. break;
  605. }
  606. }
  607. }
  608. }
  609. _uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2));
  610. _lastMultipathCompatibilityCheck = now;
  611. _localMultipathSupported = ((RR->node->getMultipathMode() != ZT_MULTIPATH_NONE) && (ZT_PROTO_VERSION > 9));
  612. _remoteMultipathSupported = _vProto > 9;
  613. // If both peers support multipath and more than one path exist, we can use multipath logic
  614. _canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1);
  615. }
  616. }
  617. void Peer::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
  618. {
  619. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ACK);
  620. uint32_t bytesToAck = path->bytesToAck();
  621. outp.append<uint32_t>(bytesToAck);
  622. if (atAddress) {
  623. outp.armor(_key,false);
  624. RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
  625. } else {
  626. RR->sw->send(tPtr,outp,false);
  627. }
  628. path->sentAck(now);
  629. }
  630. void Peer::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
  631. {
  632. const int64_t _now = RR->node->now();
  633. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_QOS_MEASUREMENT);
  634. char qosData[ZT_PATH_MAX_QOS_PACKET_SZ];
  635. int16_t len = path->generateQoSPacket(_now,qosData);
  636. outp.append(qosData,len);
  637. if (atAddress) {
  638. outp.armor(_key,false);
  639. RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
  640. } else {
  641. RR->sw->send(tPtr,outp,false);
  642. }
  643. path->sentQoS(now);
  644. }
  645. void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
  646. {
  647. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
  648. outp.append((unsigned char)ZT_PROTO_VERSION);
  649. outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
  650. outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
  651. outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
  652. outp.append(now);
  653. RR->identity.serialize(outp,false);
  654. atAddress.serialize(outp);
  655. outp.append((uint64_t)RR->topology->planetWorldId());
  656. outp.append((uint64_t)RR->topology->planetWorldTimestamp());
  657. const unsigned int startCryptedPortionAt = outp.size();
  658. std::vector<World> moons(RR->topology->moons());
  659. std::vector<uint64_t> moonsWanted(RR->topology->moonsWanted());
  660. outp.append((uint16_t)(moons.size() + moonsWanted.size()));
  661. for(std::vector<World>::const_iterator m(moons.begin());m!=moons.end();++m) {
  662. outp.append((uint8_t)m->type());
  663. outp.append((uint64_t)m->id());
  664. outp.append((uint64_t)m->timestamp());
  665. }
  666. for(std::vector<uint64_t>::const_iterator m(moonsWanted.begin());m!=moonsWanted.end();++m) {
  667. outp.append((uint8_t)World::TYPE_MOON);
  668. outp.append(*m);
  669. outp.append((uint64_t)0);
  670. }
  671. outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt);
  672. RR->node->expectReplyTo(outp.packetId());
  673. if (atAddress) {
  674. outp.armor(_key,false); // false == don't encrypt full payload, but add MAC
  675. RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
  676. } else {
  677. RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC
  678. }
  679. }
  680. void Peer::attemptToContactAt(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now,bool sendFullHello)
  681. {
  682. if ( (!sendFullHello) && (_vProto >= 5) && (!((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0))) ) {
  683. Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
  684. RR->node->expectReplyTo(outp.packetId());
  685. outp.armor(_key,true);
  686. RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
  687. } else {
  688. sendHELLO(tPtr,localSocket,atAddress,now);
  689. }
  690. }
  691. void Peer::tryMemorizedPath(void *tPtr,int64_t now)
  692. {
  693. if ((now - _lastTriedMemorizedPath) >= ZT_TRY_MEMORIZED_PATH_INTERVAL) {
  694. _lastTriedMemorizedPath = now;
  695. InetAddress mp;
  696. if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp))
  697. attemptToContactAt(tPtr,-1,mp,now,true);
  698. }
  699. }
  700. unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
  701. {
  702. unsigned int sent = 0;
  703. Mutex::Lock _l(_paths_m);
  704. const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD);
  705. _lastSentFullHello = now;
  706. processBackgroundPeerTasks(now);
  707. // Emit traces regarding aggregate link status
  708. if (_canUseMultipath) {
  709. int alivePathCount = aggregateLinkPhysicalPathCount();
  710. if ((now - _lastAggregateStatsReport) > ZT_PATH_AGGREGATE_STATS_REPORT_INTERVAL) {
  711. _lastAggregateStatsReport = now;
  712. if (alivePathCount) {
  713. RR->t->peerLinkAggregateStatistics(NULL,*this);
  714. }
  715. } if (alivePathCount < 2 && _linkIsRedundant) {
  716. _linkIsRedundant = !_linkIsRedundant;
  717. RR->t->peerLinkNoLongerRedundant(NULL,*this);
  718. } if (alivePathCount > 1 && !_linkIsRedundant) {
  719. _linkIsRedundant = !_linkIsRedundant;
  720. RR->t->peerLinkNowRedundant(NULL,*this);
  721. }
  722. }
  723. // Right now we only keep pinging links that have the maximum priority. The
  724. // priority is used to track cluster redirections, meaning that when a cluster
  725. // redirects us its redirect target links override all other links and we
  726. // let those old links expire.
  727. long maxPriority = 0;
  728. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  729. if (_paths[i].p)
  730. maxPriority = std::max(_paths[i].priority,maxPriority);
  731. else break;
  732. }
  733. unsigned int j = 0;
  734. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  735. if (_paths[i].p) {
  736. // Clean expired and reduced priority paths
  737. if ( ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) && (_paths[i].priority == maxPriority) ) {
  738. if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))) {
  739. attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello);
  740. _paths[i].p->sent(now);
  741. sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2;
  742. }
  743. if (i != j)
  744. _paths[j] = _paths[i];
  745. ++j;
  746. }
  747. } else break;
  748. }
  749. if (canUseMultipath()) {
  750. while(j < ZT_MAX_PEER_NETWORK_PATHS) {
  751. _paths[j].lr = 0;
  752. _paths[j].p.zero();
  753. _paths[j].priority = 1;
  754. ++j;
  755. }
  756. }
  757. return sent;
  758. }
  759. void Peer::clusterRedirect(void *tPtr,const SharedPtr<Path> &originatingPath,const InetAddress &remoteAddress,const int64_t now)
  760. {
  761. SharedPtr<Path> np(RR->topology->getPath(originatingPath->localSocket(),remoteAddress));
  762. RR->t->peerRedirected(tPtr,0,*this,np);
  763. attemptToContactAt(tPtr,originatingPath->localSocket(),remoteAddress,now,true);
  764. {
  765. Mutex::Lock _l(_paths_m);
  766. // New priority is higher than the priority of the originating path (if known)
  767. long newPriority = 1;
  768. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  769. if (_paths[i].p) {
  770. if (_paths[i].p == originatingPath) {
  771. newPriority = _paths[i].priority;
  772. break;
  773. }
  774. } else break;
  775. }
  776. newPriority += 2;
  777. // Erase any paths with lower priority than this one or that are duplicate
  778. // IPs and add this path.
  779. unsigned int j = 0;
  780. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  781. if (_paths[i].p) {
  782. if ((_paths[i].priority >= newPriority)&&(!_paths[i].p->address().ipsEqual2(remoteAddress))) {
  783. if (i != j)
  784. _paths[j] = _paths[i];
  785. ++j;
  786. }
  787. }
  788. }
  789. if (j < ZT_MAX_PEER_NETWORK_PATHS) {
  790. _paths[j].lr = now;
  791. _paths[j].p = np;
  792. _paths[j].priority = newPriority;
  793. ++j;
  794. while (j < ZT_MAX_PEER_NETWORK_PATHS) {
  795. _paths[j].lr = 0;
  796. _paths[j].p.zero();
  797. _paths[j].priority = 1;
  798. ++j;
  799. }
  800. }
  801. }
  802. }
  803. void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now)
  804. {
  805. Mutex::Lock _l(_paths_m);
  806. for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
  807. if (_paths[i].p) {
  808. if ((_paths[i].p->address().ss_family == inetAddressFamily)&&(_paths[i].p->ipScope() == scope)) {
  809. attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,false);
  810. _paths[i].p->sent(now);
  811. _paths[i].lr = 0; // path will not be used unless it speaks again
  812. }
  813. } else break;
  814. }
  815. }
  816. } // namespace ZeroTier