Bond.hpp 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518
  1. /*
  2. * Copyright (c)2013-2021 ZeroTier, Inc.
  3. *
  4. * Use of this software is governed by the Business Source License included
  5. * in the LICENSE.TXT file in the project's root directory.
  6. *
  7. * Change Date: 2026-01-01
  8. *
  9. * On the date above, in accordance with the Business Source License, use
  10. * of this software will be governed by version 2.0 of the Apache License.
  11. */
  12. /****/
  13. #ifndef ZT_BOND_HPP
  14. #define ZT_BOND_HPP
  15. #include "../osdep/Phy.hpp"
  16. #include "Packet.hpp"
  17. #include "Path.hpp"
  18. #include "RuntimeEnvironment.hpp"
  19. #include "Trace.hpp"
  20. #include <cstdarg>
  21. #include <deque>
  22. #include <list>
  23. #include <map>
  24. /**
  25. * Indices for the path quality weight vector
  26. */
  27. enum ZT_BondQualityWeightIndex { ZT_QOS_LAT_IDX, ZT_QOS_LTM_IDX, ZT_QOS_PDV_IDX, ZT_QOS_PLR_IDX, ZT_QOS_PER_IDX, ZT_QOS_THR_IDX, ZT_QOS_THM_IDX, ZT_QOS_THV_IDX, ZT_QOS_AGE_IDX, ZT_QOS_SCP_IDX, ZT_QOS_WEIGHT_SIZE };
  28. /**
  29. * Multipath bonding policy
  30. */
  31. enum ZT_BondBondingPolicy {
  32. /**
  33. * Normal operation. No fault tolerance, no load balancing
  34. */
  35. ZT_BOND_POLICY_NONE = 0,
  36. /**
  37. * Sends traffic out on only one path at a time. Configurable immediate
  38. * fail-over.
  39. */
  40. ZT_BOND_POLICY_ACTIVE_BACKUP = 1,
  41. /**
  42. * Sends traffic out on all paths
  43. */
  44. ZT_BOND_POLICY_BROADCAST = 2,
  45. /**
  46. * Stripes packets across all paths
  47. */
  48. ZT_BOND_POLICY_BALANCE_RR = 3,
  49. /**
  50. * Packets destined for specific peers will always be sent over the same
  51. * path.
  52. */
  53. ZT_BOND_POLICY_BALANCE_XOR = 4,
  54. /**
  55. * Balances flows among all paths according to path performance
  56. */
  57. ZT_BOND_POLICY_BALANCE_AWARE = 5
  58. };
  59. /**
  60. * Multipath active re-selection policy (linkSelectMethod)
  61. */
  62. enum ZT_BondLinkSelectMethod {
  63. /**
  64. * Primary link regains status as active link whenever it comes back up
  65. * (default when links are explicitly specified)
  66. */
  67. ZT_BOND_RESELECTION_POLICY_ALWAYS = 0,
  68. /**
  69. * Primary link regains status as active link when it comes back up and
  70. * (if) it is better than the currently-active link.
  71. */
  72. ZT_BOND_RESELECTION_POLICY_BETTER = 1,
  73. /**
  74. * Primary link regains status as active link only if the currently-active
  75. * link fails.
  76. */
  77. ZT_BOND_RESELECTION_POLICY_FAILURE = 2,
  78. /**
  79. * The primary link can change if a superior path is detected.
  80. * (default if user provides no fail-over guidance)
  81. */
  82. ZT_BOND_RESELECTION_POLICY_OPTIMIZE = 3
  83. };
  84. /**
  85. * Mode of multipath link interface
  86. */
  87. enum ZT_BondLinkMode { ZT_BOND_SLAVE_MODE_PRIMARY = 0, ZT_BOND_SLAVE_MODE_SPARE = 1 };
  88. #include "../node/AtomicCounter.hpp"
  89. #include "../node/SharedPtr.hpp"
  90. #include <string>
  91. namespace ZeroTier {
  92. class Link {
  93. friend class SharedPtr<Link>;
  94. public:
  95. /**
  96. *
  97. * @param ifnameStr
  98. * @param ipvPref
  99. * @param speed
  100. * @param enabled
  101. * @param mode
  102. * @param failoverToLinkStr
  103. * @param userSpecifiedAlloc
  104. */
  105. Link(std::string ifnameStr, uint8_t ipvPref, uint32_t speed, bool enabled, uint8_t mode, std::string failoverToLinkStr, float userSpecifiedAlloc)
  106. : _ifnameStr(ifnameStr)
  107. , _ipvPref(ipvPref)
  108. , _speed(speed)
  109. , _relativeSpeed(0)
  110. , _enabled(enabled)
  111. , _mode(mode)
  112. , _failoverToLinkStr(failoverToLinkStr)
  113. , _userSpecifiedAlloc(userSpecifiedAlloc)
  114. , _isUserSpecified(false)
  115. {
  116. }
  117. /**
  118. * @return The string representation of this link's underlying interface's system name.
  119. */
  120. inline std::string ifname()
  121. {
  122. return _ifnameStr;
  123. }
  124. /**
  125. * @return Whether this link is designated as a primary.
  126. */
  127. inline bool primary()
  128. {
  129. return _mode == ZT_BOND_SLAVE_MODE_PRIMARY;
  130. }
  131. /**
  132. * @return Whether this link is designated as a spare.
  133. */
  134. inline bool spare()
  135. {
  136. return _mode == ZT_BOND_SLAVE_MODE_SPARE;
  137. }
  138. /**
  139. * @return The name of the link interface that should be used in the event of a failure.
  140. */
  141. inline std::string failoverToLink()
  142. {
  143. return _failoverToLinkStr;
  144. }
  145. /**
  146. * @return Whether this link interface was specified by the user or auto-detected.
  147. */
  148. inline bool isUserSpecified()
  149. {
  150. return _isUserSpecified;
  151. }
  152. /**
  153. * Signify that this link was specified by the user and not the result of auto-detection.
  154. *
  155. * @param isUserSpecified
  156. */
  157. inline void setAsUserSpecified(bool isUserSpecified)
  158. {
  159. _isUserSpecified = isUserSpecified;
  160. }
  161. /**
  162. * @return Whether or not the user has specified failover instructions.
  163. */
  164. inline bool userHasSpecifiedFailoverInstructions()
  165. {
  166. return _failoverToLinkStr.length();
  167. }
  168. /**
  169. * @return The speed of the link relative to others in the bond.
  170. */
  171. inline uint8_t relativeSpeed()
  172. {
  173. return _relativeSpeed;
  174. }
  175. /**
  176. * Sets the speed of the link relative to others in the bond.
  177. *
  178. * @param relativeSpeed The speed relative to the rest of the link.
  179. */
  180. inline void setRelativeSpeed(uint8_t relativeSpeed)
  181. {
  182. _relativeSpeed = relativeSpeed;
  183. }
  184. /**
  185. * @return The absolute speed of the link (as specified by the user.)
  186. */
  187. inline uint32_t speed()
  188. {
  189. return _speed;
  190. }
  191. /**
  192. * @return The address preference for this link (as specified by the user.)
  193. */
  194. inline uint8_t ipvPref()
  195. {
  196. return _ipvPref;
  197. }
  198. /**
  199. * @return The mode (e.g. primary/spare) for this link (as specified by the user.)
  200. */
  201. inline uint8_t mode()
  202. {
  203. return _mode;
  204. }
  205. /**
  206. * @return Whether this link is enabled or disabled
  207. */
  208. inline uint8_t enabled()
  209. {
  210. return _enabled;
  211. }
  212. private:
  213. /**
  214. * String representation of underlying interface's system name
  215. */
  216. std::string _ifnameStr;
  217. /**
  218. * What preference (if any) a user has for IP protocol version used in
  219. * path aggregations. Preference is expressed in the order of the digits:
  220. *
  221. * 0: no preference
  222. * 4: IPv4 only
  223. * 6: IPv6 only
  224. * 46: IPv4 over IPv6
  225. * 64: IPv6 over IPv4
  226. */
  227. uint8_t _ipvPref;
  228. /**
  229. * User-specified speed of this link
  230. */
  231. uint32_t _speed;
  232. /**
  233. * Speed relative to other specified links (computed by Bond)
  234. */
  235. uint8_t _relativeSpeed;
  236. /**
  237. * Whether this link is enabled, or (disabled (possibly bad config))
  238. */
  239. uint8_t _enabled;
  240. /**
  241. * Whether this link is designated as a primary, a spare, or no preference.
  242. */
  243. uint8_t _mode;
  244. /**
  245. * The specific name of the link to be used in the event that this
  246. * link fails.
  247. */
  248. std::string _failoverToLinkStr;
  249. /**
  250. * User-specified allocation
  251. */
  252. float _userSpecifiedAlloc;
  253. /**
  254. * Whether or not this link was created as a result of manual user specification. This is
  255. * important to know because certain policy decisions are dependent on whether the user
  256. * intents to use a specific set of interfaces.
  257. */
  258. bool _isUserSpecified;
  259. AtomicCounter __refCount;
  260. };
  261. class Link;
  262. class Peer;
  263. class Bond {
  264. public:
  265. /**
  266. * @return Whether this link is permitted to become a member of a bond.
  267. */
  268. static bool linkAllowed(std::string& policyAlias, SharedPtr<Link> link);
  269. /**
  270. * @return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
  271. */
  272. static int minReqMonitorInterval()
  273. {
  274. return _minReqMonitorInterval;
  275. }
  276. /**
  277. * @return Whether the bonding layer is currently set up to be used.
  278. */
  279. static bool inUse()
  280. {
  281. return ! _bondPolicyTemplates.empty() || _defaultPolicy;
  282. }
  283. /**
  284. * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
  285. * @return The bonding policy code for a given human-readable bonding policy name
  286. */
  287. static int getPolicyCodeByStr(const std::string& basePolicyName)
  288. {
  289. if (basePolicyName == "active-backup") {
  290. return 1;
  291. }
  292. if (basePolicyName == "broadcast") {
  293. return 2;
  294. }
  295. if (basePolicyName == "balance-rr") {
  296. return 3;
  297. }
  298. if (basePolicyName == "balance-xor") {
  299. return 4;
  300. }
  301. if (basePolicyName == "balance-aware") {
  302. return 5;
  303. }
  304. return 0; // "none"
  305. }
  306. /**
  307. * @param policy Bonding policy code (See ZeroTierOne.h)
  308. * @return The human-readable name for the given bonding policy code
  309. */
  310. static std::string getPolicyStrByCode(int policy)
  311. {
  312. if (policy == 1) {
  313. return "active-backup";
  314. }
  315. if (policy == 2) {
  316. return "broadcast";
  317. }
  318. if (policy == 3) {
  319. return "balance-rr";
  320. }
  321. if (policy == 4) {
  322. return "balance-xor";
  323. }
  324. if (policy == 5) {
  325. return "balance-aware";
  326. }
  327. return "none";
  328. }
  329. /**
  330. * Sets the default bonding policy for new or undefined bonds.
  331. *
  332. * @param bp Bonding policy
  333. */
  334. static void setBondingLayerDefaultPolicy(uint8_t bp)
  335. {
  336. _defaultPolicy = bp;
  337. }
  338. /**
  339. * Sets the default (custom) bonding policy for new or undefined bonds.
  340. *
  341. * @param alias Human-readable string alias for bonding policy
  342. */
  343. static void setBondingLayerDefaultPolicyStr(std::string alias)
  344. {
  345. _defaultPolicyStr = alias;
  346. }
  347. /**
  348. * Add a user-defined link to a given bonding policy.
  349. *
  350. * @param policyAlias User-defined custom name for variant of bonding policy
  351. * @param link Pointer to new link definition
  352. */
  353. static void addCustomLink(std::string& policyAlias, SharedPtr<Link> link);
  354. /**
  355. * Add a user-defined bonding policy that is based on one of the standard types.
  356. *
  357. * @param newBond Pointer to custom Bond object
  358. * @return Whether a uniquely-named custom policy was successfully added
  359. */
  360. static bool addCustomPolicy(const SharedPtr<Bond>& newBond);
  361. /**
  362. * Assigns a specific bonding policy
  363. *
  364. * @param identity
  365. * @param policyAlias
  366. * @return
  367. */
  368. static bool assignBondingPolicyToPeer(int64_t identity, const std::string& policyAlias);
  369. /**
  370. * Get pointer to bond by a given peer ID
  371. *
  372. * @param peer Remote peer ID
  373. * @return A pointer to the Bond
  374. */
  375. static SharedPtr<Bond> getBondByPeerId(int64_t identity);
  376. /**
  377. * Add a new bond to the bond controller.
  378. *
  379. * @param renv Runtime environment
  380. * @param peer Remote peer that this bond services
  381. * @return A pointer to the newly created Bond
  382. */
  383. static SharedPtr<Bond> createTransportTriggeredBond(const RuntimeEnvironment* renv, const SharedPtr<Peer>& peer);
  384. /**
  385. * Periodically perform maintenance tasks for the bonding layer.
  386. *
  387. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  388. * @param now Current time
  389. */
  390. static void processBackgroundTasks(void* tPtr, int64_t now);
  391. /**
  392. * Gets a reference to a physical link definition given a policy alias and a local socket.
  393. *
  394. * @param policyAlias Policy in use
  395. * @param localSocket Local source socket
  396. * @return Physical link definition
  397. */
  398. static SharedPtr<Link> getLinkBySocket(const std::string& policyAlias, uint64_t localSocket);
  399. /**
  400. * Gets a reference to a physical link definition given its human-readable system name.
  401. *
  402. * @param policyAlias Policy in use
  403. * @param ifname Alphanumeric human-readable name
  404. * @return Physical link definition
  405. */
  406. static SharedPtr<Link> getLinkByName(const std::string& policyAlias, const std::string& ifname);
  407. private:
  408. static Phy<Bond*>* _phy;
  409. static Mutex _bonds_m;
  410. static Mutex _links_m;
  411. /**
  412. * The minimum required monitoring interval among all bonds
  413. */
  414. static int _minReqMonitorInterval;
  415. /**
  416. * The default bonding policy used for new bonds unless otherwise specified.
  417. */
  418. static uint8_t _defaultPolicy;
  419. /**
  420. * The default bonding policy used for new bonds unless otherwise specified.
  421. */
  422. static std::string _defaultPolicyStr;
  423. /**
  424. * All currently active bonds.
  425. */
  426. static std::map<int64_t, SharedPtr<Bond> > _bonds;
  427. /**
  428. * Map of peers to custom bonding policies
  429. */
  430. static std::map<int64_t, std::string> _policyTemplateAssignments;
  431. /**
  432. * User-defined bonding policies (can be assigned to a peer)
  433. */
  434. static std::map<std::string, SharedPtr<Bond> > _bondPolicyTemplates;
  435. /**
  436. * Set of links defined for a given bonding policy
  437. */
  438. static std::map<std::string, std::vector<SharedPtr<Link> > > _linkDefinitions;
  439. /**
  440. * Set of link objects mapped to their physical interfaces
  441. */
  442. static std::map<std::string, std::map<std::string, SharedPtr<Link> > > _interfaceToLinkMap;
  443. struct NominatedPath;
  444. struct Flow;
  445. friend class SharedPtr<Bond>;
  446. friend class Peer;
  447. public:
  448. void dumpInfo(int64_t now, bool force);
  449. void dumpPathStatus(int64_t now, int pathIdx);
  450. SharedPtr<Link> getLink(const SharedPtr<Path>& path);
  451. /**
  452. * Constructor
  453. *
  454. *
  455. */
  456. Bond(const RuntimeEnvironment* renv);
  457. /**
  458. * Constructor. Creates a bond based off of ZT defaults
  459. *
  460. * @param renv Runtime environment
  461. * @param policy Bonding policy
  462. * @param peer
  463. */
  464. Bond(const RuntimeEnvironment* renv, int policy, const SharedPtr<Peer>& peer);
  465. /**
  466. * Constructor. For use when user intends to manually specify parameters
  467. *
  468. * @param basePolicy
  469. * @param policyAlias
  470. * @param peer
  471. */
  472. Bond(const RuntimeEnvironment* renv, std::string& basePolicy, std::string& policyAlias, const SharedPtr<Peer>& peer);
  473. /**
  474. * Constructor. Creates a bond based off of a user-defined bond template
  475. *
  476. * @param renv Runtime environment
  477. * @param original
  478. * @param peer
  479. */
  480. Bond(const RuntimeEnvironment* renv, SharedPtr<Bond> originalBond, const SharedPtr<Peer>& peer);
  481. /**
  482. * @return The human-readable name of the bonding policy
  483. */
  484. std::string policyAlias()
  485. {
  486. return _policyAlias;
  487. }
  488. /**
  489. * Inform the bond about the path that its peer (owning object) just learned about.
  490. * If the path is allowed to be used, it will be inducted into the bond on a trial
  491. * period where link statistics will be collected to judge its quality.
  492. *
  493. * @param path Newly-learned Path which should now be handled by the Bond
  494. * @param now Current time
  495. */
  496. void nominatePathToBond(const SharedPtr<Path>& path, int64_t now);
  497. /**
  498. * Add a nominated path to the bond. This merely maps the index from the nominated set
  499. * to a smaller set and sets the path's bonded flag to true.
  500. *
  501. * @param nominatedIdx The index in the nominated set
  502. * @param bondedIdx The index in the bonded set (subset of nominated)
  503. */
  504. void addPathToBond(int nominatedIdx, int bondedIdx);
  505. /**
  506. * Check path states and perform bond rebuilds if needed.
  507. *
  508. * @param now Current time
  509. * @param rebuild Whether or not the bond should be reconstructed.
  510. */
  511. void curateBond(int64_t now, bool rebuild);
  512. /**
  513. * Periodically perform statistical summaries of quality metrics for all paths.
  514. *
  515. * @param now Current time
  516. */
  517. void estimatePathQuality(int64_t now);
  518. /**
  519. * Record an invalid incoming packet. This packet failed
  520. * MAC/compression/cipher checks and will now contribute to a
  521. * Packet Error Ratio (PER).
  522. *
  523. * @param path Path over which packet was received
  524. */
  525. void recordIncomingInvalidPacket(const SharedPtr<Path>& path);
  526. /**
  527. * Record statistics on outbound an packet.
  528. *
  529. * @param path Path over which packet is being sent
  530. * @param packetId Packet ID
  531. * @param payloadLength Packet data length
  532. * @param verb Packet verb
  533. * @param flowId Flow ID
  534. * @param now Current time
  535. */
  536. void recordOutgoingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength, Packet::Verb verb, int32_t flowId, int64_t now);
  537. /**
  538. * Process the contents of an inbound VERB_QOS_MEASUREMENT to gather path quality observations.
  539. *
  540. * @param now Current time
  541. * @param count Number of records
  542. * @param rx_id table of packet IDs
  543. * @param rx_ts table of holding times
  544. */
  545. void receivedQoS(const SharedPtr<Path>& path, int64_t now, int count, uint64_t* rx_id, uint16_t* rx_ts);
  546. /**
  547. * Generate the contents of a VERB_QOS_MEASUREMENT packet.
  548. *
  549. * @param now Current time
  550. * @param qosBuffer destination buffer
  551. * @return Size of payload
  552. */
  553. int32_t generateQoSPacket(int pathIdx, int64_t now, char* qosBuffer);
  554. /**
  555. * Record statistics for an inbound packet.
  556. *
  557. * @param path Path over which packet was received
  558. * @param packetId Packet ID
  559. * @param payloadLength Packet data length
  560. * @param verb Packet verb
  561. * @param flowId Flow ID
  562. * @param now Current time
  563. */
  564. void recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength, Packet::Verb verb, int32_t flowId, int64_t now);
  565. /**
  566. * Determines the most appropriate path for packet and flow egress. This decision is made by
  567. * the underlying bonding policy as well as QoS-related statistical observations of path quality.
  568. *
  569. * @param now Current time
  570. * @param flowId Flow ID
  571. * @return Pointer to suggested Path
  572. */
  573. SharedPtr<Path> getAppropriatePath(int64_t now, int32_t flowId);
  574. /**
  575. * Creates a new flow record
  576. *
  577. * @param np Path over which flow shall be handled
  578. * @param flowId Flow ID
  579. * @param entropy A byte of entropy to be used by the bonding algorithm
  580. * @param now Current time
  581. * @return Pointer to newly-created Flow
  582. */
  583. SharedPtr<Flow> createFlow(int pathIdx, int32_t flowId, unsigned char entropy, int64_t now);
  584. /**
  585. * Removes flow records that are past a certain age limit.
  586. *
  587. * @param age Age threshold to be forgotten
  588. * @param oldest Whether only the oldest shall be forgotten
  589. * @param now Current time
  590. */
  591. void forgetFlowsWhenNecessary(uint64_t age, bool oldest, int64_t now);
  592. /**
  593. * Assigns a new flow to a bonded path
  594. *
  595. * @param flow Flow to be assigned
  596. * @param now Current time
  597. */
  598. bool assignFlowToBondedPath(SharedPtr<Flow>& flow, int64_t now);
  599. /**
  600. * Determine whether a path change should occur given the remote peer's reported utility and our
  601. * local peer's known utility. This has the effect of assigning inbound and outbound traffic to
  602. * the same path.
  603. *
  604. * @param now Current time
  605. * @param path Path over which the negotiation request was received
  606. * @param remoteUtility How much utility the remote peer claims to gain by using the declared path
  607. */
  608. void processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path>& path, int16_t remoteUtility);
  609. /**
  610. * Determine state of path synchronization and whether a negotiation request
  611. * shall be sent to the peer.
  612. *
  613. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  614. * @param now Current time
  615. */
  616. void pathNegotiationCheck(void* tPtr, int64_t now);
  617. /**
  618. * Sends a VERB_ACK to the remote peer.
  619. *
  620. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  621. * @param path Path over which packet should be sent
  622. * @param localSocket Local source socket
  623. * @param atAddress
  624. * @param now Current time
  625. */
  626. void sendACK(void* tPtr, int pathIdx, int64_t localSocket, const InetAddress& atAddress, int64_t now);
  627. /**
  628. * Sends a VERB_QOS_MEASUREMENT to the remote peer.
  629. *
  630. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  631. * @param path Path over which packet should be sent
  632. * @param localSocket Local source socket
  633. * @param atAddress
  634. * @param now Current time
  635. */
  636. void sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, const InetAddress& atAddress, int64_t now);
  637. /**
  638. * Sends a VERB_PATH_NEGOTIATION_REQUEST to the remote peer.
  639. *
  640. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  641. * @param path Path over which packet should be sent
  642. */
  643. void sendPATH_NEGOTIATION_REQUEST(void* tPtr, int pathIdx);
  644. /**
  645. *
  646. * @param now Current time
  647. */
  648. void processBalanceTasks(int64_t now);
  649. /**
  650. * Perform periodic tasks unique to active-backup
  651. *
  652. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  653. * @param now Current time
  654. */
  655. void processActiveBackupTasks(void* tPtr, int64_t now);
  656. /**
  657. * Switches the active link in an active-backup scenario to the next best during
  658. * a failover event.
  659. *
  660. * @param now Current time
  661. */
  662. void dequeueNextActiveBackupPath(uint64_t now);
  663. /**
  664. * Set bond parameters to reasonable defaults, these may later be overwritten by
  665. * user-specified parameters.
  666. *
  667. * @param policy Bonding policy
  668. * @param templateBond
  669. */
  670. void setBondParameters(int policy, SharedPtr<Bond> templateBond, bool useTemplate);
  671. /**
  672. * Check and assign user-specified quality weights to this bond.
  673. *
  674. * @param weights Set of user-specified weights
  675. * @param len Length of weight vector
  676. */
  677. void setUserQualityWeights(float weights[], int len);
  678. /**
  679. * @param latencyInMilliseconds Maximum acceptable latency.
  680. */
  681. void setMaxAcceptableLatency(int16_t latencyInMilliseconds)
  682. {
  683. _maxAcceptableLatency = latencyInMilliseconds;
  684. }
  685. /**
  686. * @param latencyInMilliseconds Maximum acceptable (mean) latency.
  687. */
  688. void setMaxAcceptableMeanLatency(int16_t latencyInMilliseconds)
  689. {
  690. _maxAcceptableMeanLatency = latencyInMilliseconds;
  691. }
  692. /**
  693. * @param latencyVarianceInMilliseconds Maximum acceptable packet delay variance (jitter).
  694. */
  695. void setMaxAcceptablePacketDelayVariance(int16_t latencyVarianceInMilliseconds)
  696. {
  697. _maxAcceptablePacketDelayVariance = latencyVarianceInMilliseconds;
  698. }
  699. /**
  700. * @param lossRatio Maximum acceptable packet loss ratio (PLR).
  701. */
  702. void setMaxAcceptablePacketLossRatio(float lossRatio)
  703. {
  704. _maxAcceptablePacketLossRatio = lossRatio;
  705. }
  706. /**
  707. * @param errorRatio Maximum acceptable packet error ratio (PER).
  708. */
  709. void setMaxAcceptablePacketErrorRatio(float errorRatio)
  710. {
  711. _maxAcceptablePacketErrorRatio = errorRatio;
  712. }
  713. /**
  714. * @param errorRatio Maximum acceptable packet error ratio (PER).
  715. */
  716. void setMinAcceptableAllocation(float minAlloc)
  717. {
  718. _minAcceptableAllocation = (uint8_t)(minAlloc * 255);
  719. }
  720. /**
  721. * @return Whether the user has defined links for use on this bond
  722. */
  723. inline bool userHasSpecifiedLinks()
  724. {
  725. return _userHasSpecifiedLinks;
  726. }
  727. /**
  728. * @return Whether the user has defined a set of failover link(s) for this bond
  729. */
  730. inline bool userHasSpecifiedFailoverInstructions()
  731. {
  732. return _userHasSpecifiedFailoverInstructions;
  733. };
  734. /**
  735. * @return Whether the user has specified a primary link
  736. */
  737. inline bool userHasSpecifiedPrimaryLink()
  738. {
  739. return _userHasSpecifiedPrimaryLink;
  740. }
  741. /**
  742. * @return Whether the user has specified link speeds
  743. */
  744. inline bool userHasSpecifiedLinkSpeeds()
  745. {
  746. return _userHasSpecifiedLinkSpeeds;
  747. }
  748. /**
  749. * Periodically perform maintenance tasks for each active bond.
  750. *
  751. * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
  752. * @param now Current time
  753. */
  754. void processBackgroundBondTasks(void* tPtr, int64_t now);
  755. /**
  756. * Rate limit gate for VERB_QOS_MEASUREMENT
  757. *
  758. * @param now Current time
  759. * @return Whether the incoming packet should be rate-gated
  760. */
  761. inline bool rateGateQoS(int64_t now, SharedPtr<Path>& path)
  762. {
  763. // TODO: Verify before production
  764. char pathStr[64] = { 0 };
  765. path->address().toString(pathStr);
  766. int diff = now - _lastQoSRateCheck;
  767. if ((diff) <= (_qosSendInterval / ZT_MAX_PEER_NETWORK_PATHS)) {
  768. ++_qosCutoffCount;
  769. }
  770. else {
  771. _qosCutoffCount = 0;
  772. }
  773. _lastQoSRateCheck = now;
  774. // fprintf(stderr, "rateGateQoS (count=%d, send_interval=%d, diff=%d, path=%s)\n", _qosCutoffCount, _qosSendInterval, diff, pathStr);
  775. return (_qosCutoffCount < (ZT_MAX_PEER_NETWORK_PATHS * 2));
  776. }
  777. /**
  778. * Rate limit gate for VERB_PATH_NEGOTIATION_REQUEST
  779. *
  780. * @param now Current time
  781. * @return Whether the incoming packet should be rate-gated
  782. */
  783. inline bool rateGatePathNegotiation(int64_t now, SharedPtr<Path>& path)
  784. {
  785. // TODO: Verify before production
  786. char pathStr[64] = { 0 };
  787. path->address().toString(pathStr);
  788. int diff = now - _lastPathNegotiationReceived;
  789. if ((diff) <= (ZT_PATH_NEGOTIATION_CUTOFF_TIME / ZT_MAX_PEER_NETWORK_PATHS)) {
  790. ++_pathNegotiationCutoffCount;
  791. }
  792. else {
  793. _pathNegotiationCutoffCount = 0;
  794. }
  795. _lastPathNegotiationReceived = now;
  796. // fprintf(stderr, "rateGateNeg (count=%d, send_interval=%d, diff=%d, path=%s)\n", _pathNegotiationCutoffCount, (ZT_PATH_NEGOTIATION_CUTOFF_TIME / ZT_MAX_PEER_NETWORK_PATHS), diff, pathStr);
  797. return (_pathNegotiationCutoffCount < (ZT_MAX_PEER_NETWORK_PATHS * 2));
  798. }
  799. /**
  800. * @param interval Maximum amount of time user expects a failover to take on this bond.
  801. */
  802. inline void setFailoverInterval(uint32_t interval)
  803. {
  804. _failoverInterval = interval;
  805. }
  806. /**
  807. * @param interval Maximum amount of time user expects a failover to take on this bond.
  808. */
  809. inline uint32_t getFailoverInterval()
  810. {
  811. return _failoverInterval;
  812. }
  813. /**
  814. * @param strategy Strategy that the bond uses to re-assign protocol flows.
  815. */
  816. inline void setFlowRebalanceStrategy(uint32_t strategy)
  817. {
  818. _flowRebalanceStrategy = strategy;
  819. }
  820. /**
  821. * @param strategy Strategy that the bond uses to prob for path aliveness and quality
  822. */
  823. inline void setLinkMonitorStrategy(uint8_t strategy)
  824. {
  825. _linkMonitorStrategy = strategy;
  826. }
  827. /**
  828. * @return the current up delay parameter
  829. */
  830. inline uint16_t getUpDelay()
  831. {
  832. return _upDelay;
  833. }
  834. /**
  835. * @param upDelay Length of time before a newly-discovered path is admitted to the bond
  836. */
  837. inline void setUpDelay(int upDelay)
  838. {
  839. if (upDelay >= 0) {
  840. _upDelay = upDelay;
  841. }
  842. }
  843. /**
  844. * @return Length of time before a newly-failed path is removed from the bond
  845. */
  846. inline uint16_t getDownDelay()
  847. {
  848. return _downDelay;
  849. }
  850. /**
  851. * @param downDelay Length of time before a newly-failed path is removed from the bond
  852. */
  853. inline void setDownDelay(int downDelay)
  854. {
  855. if (downDelay >= 0) {
  856. _downDelay = downDelay;
  857. }
  858. }
  859. /**
  860. * @return The current monitoring interval for the bond
  861. */
  862. inline int monitorInterval()
  863. {
  864. return _monitorInterval;
  865. }
  866. /**
  867. * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain links.)
  868. *
  869. * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
  870. */
  871. inline void setBondMonitorInterval(uint16_t interval)
  872. {
  873. _monitorInterval = interval;
  874. }
  875. /**
  876. * @param policy Bonding policy for this bond
  877. */
  878. /*
  879. inline void setPolicy(uint8_t policy)
  880. {
  881. _policy = policy;
  882. }
  883. */
  884. /**
  885. * @return the current bonding policy
  886. */
  887. inline uint8_t policy()
  888. {
  889. return _policy;
  890. }
  891. /**
  892. * @return the health status of the bond
  893. */
  894. inline bool isHealthy()
  895. {
  896. return _isHealthy;
  897. }
  898. /**
  899. * @return the number of links comprising this bond which are considered alive
  900. */
  901. inline uint8_t getNumAliveLinks()
  902. {
  903. return _numAliveLinks;
  904. };
  905. /**
  906. * @return the number of links comprising this bond
  907. */
  908. inline uint8_t getNumTotalLinks()
  909. {
  910. return _numTotalLinks;
  911. }
  912. /**
  913. *
  914. * @param allowFlowHashing
  915. */
  916. inline void setFlowHashing(bool allowFlowHashing)
  917. {
  918. _allowFlowHashing = allowFlowHashing;
  919. }
  920. /**
  921. * @return Whether flow-hashing is currently enabled for this bond.
  922. */
  923. bool flowHashingEnabled()
  924. {
  925. return _allowFlowHashing;
  926. }
  927. /**
  928. *
  929. * @param packetsPerLink
  930. */
  931. inline void setPacketsPerLink(int packetsPerLink)
  932. {
  933. _packetsPerLink = packetsPerLink;
  934. }
  935. /**
  936. * @return Number of packets to be sent on each interface in a balance-rr bond
  937. */
  938. inline int getPacketsPerLink()
  939. {
  940. return _packetsPerLink;
  941. }
  942. /**
  943. *
  944. * @param linkSelectMethod
  945. */
  946. inline void setLinkSelectMethod(uint8_t method)
  947. {
  948. _abLinkSelectMethod = method;
  949. }
  950. /**
  951. *
  952. * @return
  953. */
  954. inline uint8_t getLinkSelectMethod()
  955. {
  956. return _abLinkSelectMethod;
  957. }
  958. /**
  959. *
  960. * @param allowPathNegotiation
  961. */
  962. inline void setAllowPathNegotiation(bool allowPathNegotiation)
  963. {
  964. _allowPathNegotiation = allowPathNegotiation;
  965. }
  966. /**
  967. *
  968. * @return
  969. */
  970. inline bool allowPathNegotiation()
  971. {
  972. return _allowPathNegotiation;
  973. }
  974. /**
  975. * Forcibly rotates the currently active link used in an active-backup bond to the next link in the failover queue
  976. *
  977. * @return True if this operation succeeded, false if otherwise
  978. */
  979. bool abForciblyRotateLink();
  980. /**
  981. * @param now Current time
  982. * @return All known paths to this peer
  983. */
  984. inline std::vector<SharedPtr<Path> > paths(const int64_t now) const
  985. {
  986. std::vector<SharedPtr<Path> > pp;
  987. Mutex::Lock _l(_paths_m);
  988. for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
  989. if (! _paths[i].p)
  990. break;
  991. pp.push_back(_paths[i].p);
  992. }
  993. return pp;
  994. }
  995. /**
  996. * Emit message to tracing system but with added timestamp and subsystem info
  997. *
  998. * TODO: Will be replaced when better logging facilities exist in Trace.hpp
  999. */
  1000. void log(const char* fmt, ...)
  1001. {
  1002. // TODO: remove
  1003. if (_peerId != 0xe421efd4b9 && _peerId != 0xa03bcb43bb) {
  1004. return;
  1005. }
  1006. time_t rawtime;
  1007. struct tm* timeinfo;
  1008. char timestamp[80];
  1009. time(&rawtime);
  1010. timeinfo = localtime(&rawtime);
  1011. strftime(timestamp, 80, "%F %T", timeinfo);
  1012. #define MAX_BOND_MSG_LEN 1024
  1013. char traceMsg[MAX_BOND_MSG_LEN];
  1014. char userMsg[MAX_BOND_MSG_LEN];
  1015. va_list args;
  1016. va_start(args, fmt);
  1017. if (vsnprintf(userMsg, sizeof(userMsg), fmt, args) < 0) {
  1018. fprintf(stderr, "Encountered format encoding error while writing to trace log\n");
  1019. return;
  1020. }
  1021. snprintf(traceMsg, MAX_BOND_MSG_LEN, "%s (%llx/%s) %s", timestamp, _peerId, _policyAlias.c_str(), userMsg);
  1022. va_end(args);
  1023. RR->t->bondStateMessage(NULL, traceMsg);
  1024. #undef MAX_MSG_LEN
  1025. }
  1026. private:
  1027. struct NominatedPath {
  1028. NominatedPath()
  1029. : lastQoSMeasurement(0)
  1030. , lastThroughputEstimation(0)
  1031. , lastRefractoryUpdate(0)
  1032. , lastAliveToggle(0)
  1033. , alive(false)
  1034. , eligible(true)
  1035. , whenNominated(0)
  1036. , refractoryPeriod(0)
  1037. , ipvPref(0)
  1038. , mode(0)
  1039. , onlyPathOnLink(false)
  1040. , bonded(false)
  1041. , negotiated(false)
  1042. , shouldReallocateFlows(false)
  1043. , assignedFlowCount(0)
  1044. , latencyMean(0)
  1045. , latencyVariance(0)
  1046. , packetLossRatio(0)
  1047. , packetErrorRatio(0)
  1048. , throughputMean(0)
  1049. , throughputMax(0)
  1050. , throughputVariance(0)
  1051. , allocation(0)
  1052. , byteLoad(0)
  1053. , relativeByteLoad(0)
  1054. , affinity(0)
  1055. , failoverScore(0)
  1056. , packetsReceivedSinceLastQoS(0)
  1057. , packetsIn(0)
  1058. , packetsOut(0)
  1059. {
  1060. }
  1061. /**
  1062. * Set or update a refractory period for the path.
  1063. *
  1064. * @param punishment How much a path should be punished
  1065. * @param pathFailure Whether this call is the result of a recent path failure
  1066. */
  1067. inline void adjustRefractoryPeriod(int64_t now, uint32_t punishment, bool pathFailure)
  1068. {
  1069. if (pathFailure) {
  1070. unsigned int suggestedRefractoryPeriod = refractoryPeriod ? punishment + (refractoryPeriod * 2) : punishment;
  1071. refractoryPeriod = std::min(suggestedRefractoryPeriod, (unsigned int)ZT_BOND_MAX_REFRACTORY_PERIOD);
  1072. lastRefractoryUpdate = 0;
  1073. }
  1074. else {
  1075. uint32_t drainRefractory = 0;
  1076. if (lastRefractoryUpdate) {
  1077. drainRefractory = (now - lastRefractoryUpdate);
  1078. }
  1079. else {
  1080. drainRefractory = (now - lastAliveToggle);
  1081. }
  1082. lastRefractoryUpdate = now;
  1083. if (refractoryPeriod > drainRefractory) {
  1084. refractoryPeriod -= drainRefractory;
  1085. }
  1086. else {
  1087. refractoryPeriod = 0;
  1088. lastRefractoryUpdate = 0;
  1089. }
  1090. }
  1091. }
  1092. /**
  1093. * @return True if a path is permitted to be used in a bond (according to user pref.)
  1094. */
  1095. inline bool allowed()
  1096. {
  1097. return (! ipvPref || ((p->_addr.isV4() && (ipvPref == 4 || ipvPref == 46 || ipvPref == 64)) || ((p->_addr.isV6() && (ipvPref == 6 || ipvPref == 46 || ipvPref == 64)))));
  1098. }
  1099. /**
  1100. * @return True if a path is preferred over another on the same physical link (according to user pref.)
  1101. */
  1102. inline bool preferred()
  1103. {
  1104. return onlyPathOnLink || (p->_addr.isV4() && (ipvPref == 4 || ipvPref == 46)) || (p->_addr.isV6() && (ipvPref == 6 || ipvPref == 64));
  1105. }
  1106. /**
  1107. * @param now Current time
  1108. * @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time
  1109. */
  1110. inline bool needsToSendQoS(int64_t now, int qosSendInterval)
  1111. {
  1112. // fprintf(stderr, "QOS table (%d / %d)\n", packetsReceivedSinceLastQoS, ZT_QOS_TABLE_SIZE);
  1113. return ((packetsReceivedSinceLastQoS >= ZT_QOS_TABLE_SIZE) || ((now - lastQoSMeasurement) > qosSendInterval)) && packetsReceivedSinceLastQoS;
  1114. }
  1115. /**
  1116. * Reset packet counters
  1117. */
  1118. inline void resetPacketCounts()
  1119. {
  1120. packetsIn = 0;
  1121. packetsOut = 0;
  1122. }
  1123. std::map<uint64_t, uint64_t> qosStatsOut; // id:egress_time
  1124. std::map<uint64_t, uint64_t> qosStatsIn; // id:now
  1125. RingBuffer<int, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordSize;
  1126. RingBuffer<float, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordLossSamples;
  1127. RingBuffer<uint64_t, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> throughputSamples;
  1128. RingBuffer<bool, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> packetValiditySamples;
  1129. RingBuffer<float, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> throughputVarianceSamples;
  1130. RingBuffer<uint16_t, ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> latencySamples;
  1131. uint64_t lastQoSMeasurement; // Last time that a VERB_QOS_MEASUREMENT was sent out on this path.
  1132. uint64_t lastThroughputEstimation; // Last time that the path's throughput was estimated.
  1133. uint64_t lastRefractoryUpdate; // The last time that the refractory period was updated.
  1134. uint64_t lastAliveToggle; // The last time that the path was marked as "alive".
  1135. bool alive;
  1136. bool eligible; // State of eligibility at last check. Used for determining state changes.
  1137. uint64_t whenNominated; // Timestamp indicating when this path's trial period began.
  1138. uint32_t refractoryPeriod; // Amount of time that this path will be prevented from becoming a member of a bond.
  1139. uint8_t ipvPref; // IP version preference inherited from the physical link.
  1140. uint8_t mode; // Mode inherited from the physical link.
  1141. bool onlyPathOnLink; // IP version preference inherited from the physical link.
  1142. bool enabled; // Enabled state inherited from the physical link.
  1143. bool bonded; // Whether this path is currently part of a bond.
  1144. bool negotiated; // Whether this path was intentionally negotiated by either peer.
  1145. bool shouldReallocateFlows; // Whether flows should be moved from this path. Current traffic flows will be re-allocated immediately.
  1146. uint16_t assignedFlowCount; // The number of flows currently assigned to this path.
  1147. float latencyMean; // The mean latency (computed from a sliding window.)
  1148. float latencyVariance; // Packet delay variance (computed from a sliding window.)
  1149. float packetLossRatio; // The ratio of lost packets to received packets.
  1150. float packetErrorRatio; // The ratio of packets that failed their MAC/CRC checks to those that did not.
  1151. uint64_t throughputMean; // The estimated mean throughput of this path.
  1152. uint64_t throughputMax; // The maximum observed throughput of this path.
  1153. float throughputVariance; // The variance in the estimated throughput of this path.
  1154. uint8_t allocation; // The relative quality of this path to all others in the bond, [0-255].
  1155. uint64_t byteLoad; // How much load this path is under.
  1156. uint8_t relativeByteLoad; // How much load this path is under (relative to other paths in the bond.)
  1157. uint8_t affinity; // Relative value expressing how "deserving" this path is of new traffic.
  1158. uint32_t failoverScore; // Score that indicates to what degree this path is preferred over others that are available to the bonding policy. (specifically for active-backup)
  1159. int32_t packetsReceivedSinceLastQoS; // Number of packets received since the last VERB_QOS_MEASUREMENT was sent to the remote peer.
  1160. /**
  1161. * Counters used for tracking path load.
  1162. */
  1163. int packetsIn;
  1164. int packetsOut;
  1165. AtomicCounter __refCount;
  1166. SharedPtr<Path> p;
  1167. void set(uint64_t now, const SharedPtr<Path>& path)
  1168. {
  1169. p = path;
  1170. whenNominated = now;
  1171. p->_bondingMetricPtr = (void*)this;
  1172. }
  1173. };
  1174. /**
  1175. * Paths nominated to the bond (may or may not actually be bonded)
  1176. */
  1177. NominatedPath _paths[ZT_MAX_PEER_NETWORK_PATHS];
  1178. inline int getNominatedPathIdx(const SharedPtr<Path>& path)
  1179. {
  1180. for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
  1181. if (_paths[i].p == path) {
  1182. return i;
  1183. }
  1184. }
  1185. return ZT_MAX_PEER_NETWORK_PATHS;
  1186. }
  1187. /**
  1188. * A protocol flow that is identified by the origin and destination port.
  1189. */
  1190. struct Flow {
  1191. /**
  1192. * @param flowId Given flow ID
  1193. * @param now Current time
  1194. */
  1195. Flow(int32_t flowId, int64_t now) : id(flowId), bytesIn(0), bytesOut(0), lastActivity(now), lastPathReassignment(0), assignedPath(ZT_MAX_PEER_NETWORK_PATHS)
  1196. {
  1197. }
  1198. /**
  1199. * Reset flow statistics
  1200. */
  1201. inline void resetByteCounts()
  1202. {
  1203. bytesIn = 0;
  1204. bytesOut = 0;
  1205. }
  1206. /**
  1207. * How long since a packet was sent or received in this flow
  1208. *
  1209. * @param now Current time
  1210. * @return The age of the flow in terms of last recorded activity
  1211. */
  1212. int64_t age(int64_t now)
  1213. {
  1214. return now - lastActivity;
  1215. }
  1216. /**
  1217. * @param path Assigned path over which this flow should be handled
  1218. */
  1219. inline void assignPath(int pathIdx, int64_t now)
  1220. {
  1221. assignedPath = pathIdx;
  1222. lastPathReassignment = now;
  1223. }
  1224. AtomicCounter __refCount;
  1225. int32_t id; // Flow ID used for hashing and path selection
  1226. uint64_t bytesIn; // Used for tracking flow size
  1227. uint64_t bytesOut; // Used for tracking flow size
  1228. int64_t lastActivity; // The last time that this flow handled traffic
  1229. int64_t lastPathReassignment; // Time of last path assignment. Used for anti-flapping
  1230. int assignedPath; // Index of path to which this flow is assigned
  1231. };
  1232. const RuntimeEnvironment* RR;
  1233. AtomicCounter __refCount;
  1234. std::string _policyAlias; // Custom name given by the user to this bond type.
  1235. /**
  1236. * Set of indices corresponding to paths currently included in the bond proper. This
  1237. * may only be updated during a call to curateBond(). The reason for this is so that
  1238. * we can simplify the high frequency packet egress logic.
  1239. */
  1240. int _bondIdxMap[ZT_MAX_PEER_NETWORK_PATHS];
  1241. int _numBondedPaths; // Number of paths currently included in the _bondIdxMap set.
  1242. std::map<int32_t, SharedPtr<Flow> > _flows; // Flows hashed according to port and protocol
  1243. float _qw[ZT_QOS_WEIGHT_SIZE]; // How much each factor contributes to the "quality" score of a path.
  1244. uint8_t _policy;
  1245. uint32_t _upDelay;
  1246. uint32_t _downDelay;
  1247. // active-backup
  1248. int _abPathIdx; // current active path
  1249. std::deque<int> _abFailoverQueue;
  1250. uint8_t _abLinkSelectMethod; // link re-selection policy for the primary link in active-backup
  1251. // balance-rr
  1252. uint8_t _rrIdx; // index to path currently in use during Round Robin operation
  1253. uint16_t _rrPacketsSentOnCurrLink; // number of packets sent on this link since the most recent path switch.
  1254. /**
  1255. * How many packets will be sent on a path before moving to the next path
  1256. * in the round-robin sequence. A value of zero will cause a random path
  1257. * selection for each outgoing packet.
  1258. */
  1259. int _packetsPerLink;
  1260. // balance-aware
  1261. uint64_t _totalBondUnderload;
  1262. uint8_t _flowRebalanceStrategy;
  1263. // dynamic link monitoring
  1264. uint8_t _linkMonitorStrategy;
  1265. // path negotiation
  1266. int16_t _localUtility;
  1267. int negotiatedPathIdx;
  1268. uint8_t _numSentPathNegotiationRequests;
  1269. bool _allowPathNegotiation;
  1270. /**
  1271. * Timers and intervals
  1272. */
  1273. uint32_t _failoverInterval;
  1274. uint32_t _qosSendInterval;
  1275. uint32_t _ackSendInterval;
  1276. uint32_t throughputMeasurementInterval;
  1277. uint32_t _qualityEstimationInterval;
  1278. /**
  1279. * Acceptable quality thresholds
  1280. */
  1281. float _maxAcceptablePacketLossRatio;
  1282. float _maxAcceptablePacketErrorRatio;
  1283. uint16_t _maxAcceptableLatency;
  1284. uint16_t _maxAcceptableMeanLatency;
  1285. uint16_t _maxAcceptablePacketDelayVariance;
  1286. uint8_t _minAcceptableAllocation;
  1287. /**
  1288. * Link state reporting
  1289. */
  1290. bool _isHealthy;
  1291. uint8_t _numAliveLinks;
  1292. uint8_t _numTotalLinks;
  1293. /**
  1294. * Default initial punishment inflicted on misbehaving paths. Punishment slowly
  1295. * drains linearly. For each eligibility change the remaining punishment is doubled.
  1296. */
  1297. uint32_t _defaultPathRefractoryPeriod;
  1298. unsigned char _freeRandomByte; // Free byte of entropy that is updated on every packet egress event.
  1299. SharedPtr<Peer> _peer; // Remote peer that this bond services
  1300. unsigned long long _peerId; // ID of the peer that this bond services
  1301. /**
  1302. * Rate-limiting
  1303. */
  1304. uint16_t _qosCutoffCount;
  1305. uint64_t _lastQoSRateCheck;
  1306. uint16_t _pathNegotiationCutoffCount;
  1307. uint64_t _lastPathNegotiationReceived;
  1308. /**
  1309. * Recent event timestamps
  1310. */
  1311. uint64_t _lastSummaryDump;
  1312. uint64_t _lastQualityEstimation;
  1313. uint64_t _lastBackgroundTaskCheck;
  1314. uint64_t _lastBondStatusLog;
  1315. uint64_t _lastPathNegotiationCheck;
  1316. uint64_t _lastSentPathNegotiationRequest;
  1317. uint64_t _lastFlowExpirationCheck;
  1318. uint64_t _lastFlowRebalance;
  1319. uint64_t _lastFrame;
  1320. uint64_t _lastActiveBackupPathChange;
  1321. Mutex _paths_m;
  1322. Mutex _flows_m;
  1323. bool _userHasSpecifiedLinks; // Whether the user has specified links for this bond.
  1324. bool _userHasSpecifiedPrimaryLink; // Whether the user has specified a primary link for this bond.
  1325. bool _userHasSpecifiedFailoverInstructions; // Whether the user has specified failover instructions for this bond.
  1326. bool _userHasSpecifiedLinkSpeeds; // Whether the user has specified links speeds for this bond.
  1327. /**
  1328. * How frequently (in ms) a VERB_ECHO is sent to a peer to verify that a
  1329. * path is still active. A value of zero (0) will disable active path
  1330. * monitoring; as result, all monitoring will be a function of traffic.
  1331. */
  1332. int _monitorInterval;
  1333. bool _allowFlowHashing; // Whether or not flow hashing is allowed.
  1334. uint64_t _overheadBytes;
  1335. };
  1336. } // namespace ZeroTier
  1337. #endif