add configuration options for HandshakeManager (#179)

This change exposes the current constants we have defined for the handshake
manager as configuration options. This will allow us to test and tweak
with different intervals and wait rotations.

    # Handshake Manger Settings
    handshakes:
      # Total time to try a handshake = sequence of `try_interval * retries`
      # With 100ms interval and 20 retries it is 23.5 seconds
      try_interval: 100ms
      retries: 20

      # wait_rotation is the number of handshake attempts to do before starting to try non-local IP addresses
      wait_rotation: 5

connection_manager_test.go

@@ -36,7 +36,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 		certState:        cs,
 		firewall:         &Firewall{},
 		lightHouse:       lh,
-		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}),
+		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}, defaultHandshakeConfig),
 	}
 	now := time.Now()
 
@@ -99,7 +99,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 		certState:        cs,
 		firewall:         &Firewall{},
 		lightHouse:       lh,
-		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}),
+		handshakeManager: NewHandshakeManager(vpncidr, preferredRanges, hostMap, lh, &udpConn{}, defaultHandshakeConfig),
 	}
 	now := time.Now()
 

examples/config.yml

@@ -139,6 +139,15 @@ logging:
   #subsystem: nebula
   #interval: 10s
 
+# Handshake Manger Settings
+#handshakes:
+  # Total time to try a handshake = sequence of `try_interval * retries`
+  # With 100ms interval and 20 retries it is 23.5 seconds
+  #try_interval: 100ms
+  #retries: 20
+  # wait_rotation is the number of handshake attempts to do before starting to try non-local IP addresses
+  #wait_rotation: 5
+
 # Nebula security group configuration
 firewall:
   conntrack:

handshake_manager.go

@@ -13,36 +13,53 @@ import (
 const (
 	// Total time to try a handshake = sequence of HandshakeTryInterval * HandshakeRetries
 	// With 100ms interval and 20 retries is 23.5 seconds
-	HandshakeTryInterval = time.Millisecond * 100
-	HandshakeRetries     = 20
-	// HandshakeWaitRotation is the number of handshake attempts to do before starting to use other ips addresses
-	HandshakeWaitRotation = 5
+	DefaultHandshakeTryInterval = time.Millisecond * 100
+	DefaultHandshakeRetries     = 20
+	// DefaultHandshakeWaitRotation is the number of handshake attempts to do before starting to use other ips addresses
+	DefaultHandshakeWaitRotation = 5
 )
 
+var (
+	defaultHandshakeConfig = HandshakeConfig{
+		tryInterval:  DefaultHandshakeTryInterval,
+		retries:      DefaultHandshakeRetries,
+		waitRotation: DefaultHandshakeWaitRotation,
+	}
+)
+
+type HandshakeConfig struct {
+	tryInterval  time.Duration
+	retries      int
+	waitRotation int
+}
+
 type HandshakeManager struct {
 	pendingHostMap *HostMap
 	mainHostMap    *HostMap
 	lightHouse     *LightHouse
 	outside        *udpConn
+	config         HandshakeConfig
 
 	OutboundHandshakeTimer *SystemTimerWheel
 	InboundHandshakeTimer  *SystemTimerWheel
 }
 
-func NewHandshakeManager(tunCidr *net.IPNet, preferredRanges []*net.IPNet, mainHostMap *HostMap, lightHouse *LightHouse, outside *udpConn) *HandshakeManager {
+func NewHandshakeManager(tunCidr *net.IPNet, preferredRanges []*net.IPNet, mainHostMap *HostMap, lightHouse *LightHouse, outside *udpConn, config HandshakeConfig) *HandshakeManager {
 	return &HandshakeManager{
 		pendingHostMap: NewHostMap("pending", tunCidr, preferredRanges),
 		mainHostMap:    mainHostMap,
 		lightHouse:     lightHouse,
 		outside:        outside,
 
-		OutboundHandshakeTimer: NewSystemTimerWheel(HandshakeTryInterval, HandshakeTryInterval*HandshakeRetries),
-		InboundHandshakeTimer:  NewSystemTimerWheel(HandshakeTryInterval, HandshakeTryInterval*HandshakeRetries),
+		config: config,
+
+		OutboundHandshakeTimer: NewSystemTimerWheel(config.tryInterval, config.tryInterval*time.Duration(config.retries)),
+		InboundHandshakeTimer:  NewSystemTimerWheel(config.tryInterval, config.tryInterval*time.Duration(config.retries)),
 	}
 }
 
 func (c *HandshakeManager) Run(f EncWriter) {
-	clockSource := time.Tick(HandshakeTryInterval)
+	clockSource := time.Tick(c.config.tryInterval)
 	for now := range clockSource {
 		c.NextOutboundHandshakeTimerTick(now, f)
 		c.NextInboundHandshakeTimerTick(now)
@@ -70,7 +87,7 @@ func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f EncWr
 
 		// If we haven't finished the handshake and we haven't hit max retries, query
 		// lighthouse and then send the handshake packet again.
-		if hostinfo.HandshakeCounter < HandshakeRetries && !hostinfo.HandshakeComplete {
+		if hostinfo.HandshakeCounter < c.config.retries && !hostinfo.HandshakeComplete {
 			if hostinfo.remote == nil {
 				// We continue to query the lighthouse because hosts may
 				// come online during handshake retries. If the query
@@ -88,7 +105,7 @@ func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f EncWr
 
 			// We want to use the "best" calculated ip for the first 5 attempts, after that we just blindly rotate through
 			// all the others until we can stand up a connection.
-			if hostinfo.HandshakeCounter > HandshakeWaitRotation {
+			if hostinfo.HandshakeCounter > c.config.waitRotation {
 				hostinfo.rotateRemote()
 			}
 
@@ -114,7 +131,7 @@ func (c *HandshakeManager) NextOutboundHandshakeTimerTick(now time.Time, f EncWr
 
 			// Readd to the timer wheel so we continue trying wait HandshakeTryInterval * counter longer for next try
 			//l.Infoln("Interval: ", HandshakeTryInterval*time.Duration(hostinfo.HandshakeCounter))
-			c.OutboundHandshakeTimer.Add(vpnIP, HandshakeTryInterval*time.Duration(hostinfo.HandshakeCounter))
+			c.OutboundHandshakeTimer.Add(vpnIP, c.config.tryInterval*time.Duration(hostinfo.HandshakeCounter))
 		} else {
 			c.pendingHostMap.DeleteVpnIP(vpnIP)
 			c.pendingHostMap.DeleteIndex(index)
@@ -144,7 +161,7 @@ func (c *HandshakeManager) AddVpnIP(vpnIP uint32) *HostInfo {
 	hostinfo := c.pendingHostMap.AddVpnIP(vpnIP)
 	// We lock here and use an array to insert items to prevent locking the
 	// main receive thread for very long by waiting to add items to the pending map
-	c.OutboundHandshakeTimer.Add(vpnIP, HandshakeTryInterval)
+	c.OutboundHandshakeTimer.Add(vpnIP, c.config.tryInterval)
 	return hostinfo
 }
 

handshake_manager_test.go

@@ -21,7 +21,7 @@ func Test_NewHandshakeManagerIndex(t *testing.T) {
 	preferredRanges := []*net.IPNet{localrange}
 	mainHM := NewHostMap("test", vpncidr, preferredRanges)
 
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{}, defaultHandshakeConfig)
 
 	now := time.Now()
 	blah.NextInboundHandshakeTimerTick(now)
@@ -37,8 +37,8 @@ func Test_NewHandshakeManagerIndex(t *testing.T) {
 	// Adding something to pending should not affect the main hostmap
 	assert.Len(t, mainHM.Indexes, 0)
 	// Jump ahead 8 seconds
-	for i := 1; i <= HandshakeRetries; i++ {
-		next_tick := now.Add(HandshakeTryInterval * time.Duration(i))
+	for i := 1; i <= DefaultHandshakeRetries; i++ {
+		next_tick := now.Add(DefaultHandshakeTryInterval * time.Duration(i))
 		blah.NextInboundHandshakeTimerTick(next_tick)
 	}
 	// Confirm they are still in the pending index list
@@ -63,7 +63,7 @@ func Test_NewHandshakeManagerVpnIP(t *testing.T) {
 	mw := &mockEncWriter{}
 	mainHM := NewHostMap("test", vpncidr, preferredRanges)
 
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{}, defaultHandshakeConfig)
 
 	now := time.Now()
 	blah.NextOutboundHandshakeTimerTick(now, mw)
@@ -81,8 +81,8 @@ func Test_NewHandshakeManagerVpnIP(t *testing.T) {
 
 	// Jump ahead `HandshakeRetries` ticks
 	cumulative := time.Duration(0)
-	for i := 0; i <= HandshakeRetries+1; i++ {
-		cumulative += time.Duration(i)*HandshakeTryInterval + 1
+	for i := 0; i <= DefaultHandshakeRetries+1; i++ {
+		cumulative += time.Duration(i)*DefaultHandshakeTryInterval + 1
 		next_tick := now.Add(cumulative)
 		//l.Infoln(next_tick)
 		blah.NextOutboundHandshakeTimerTick(next_tick, mw)
@@ -93,7 +93,7 @@ func Test_NewHandshakeManagerVpnIP(t *testing.T) {
 		assert.Contains(t, blah.pendingHostMap.Hosts, uint32(v))
 	}
 	// Jump ahead 1 more second
-	cumulative += time.Duration(HandshakeRetries+1) * HandshakeTryInterval
+	cumulative += time.Duration(DefaultHandshakeRetries+1) * DefaultHandshakeTryInterval
 	next_tick := now.Add(cumulative)
 	//l.Infoln(next_tick)
 	blah.NextOutboundHandshakeTimerTick(next_tick, mw)
@@ -112,7 +112,7 @@ func Test_NewHandshakeManagerVpnIPcleanup(t *testing.T) {
 	mw := &mockEncWriter{}
 	mainHM := NewHostMap("test", vpncidr, preferredRanges)
 
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{}, defaultHandshakeConfig)
 
 	now := time.Now()
 	blah.NextOutboundHandshakeTimerTick(now, mw)
@@ -125,8 +125,8 @@ func Test_NewHandshakeManagerVpnIPcleanup(t *testing.T) {
 	// Jump ahead `HandshakeRetries` ticks. Eviction should happen in pending
 	// but not main hostmap
 	cumulative := time.Duration(0)
-	for i := 1; i <= HandshakeRetries+2; i++ {
-		cumulative += HandshakeTryInterval * time.Duration(i)
+	for i := 1; i <= DefaultHandshakeRetries+2; i++ {
+		cumulative += DefaultHandshakeTryInterval * time.Duration(i)
 		next_tick := now.Add(cumulative)
 		blah.NextOutboundHandshakeTimerTick(next_tick, mw)
 	}
@@ -161,7 +161,7 @@ func Test_NewHandshakeManagerIndexcleanup(t *testing.T) {
 	preferredRanges := []*net.IPNet{localrange}
 	mainHM := NewHostMap("test", vpncidr, preferredRanges)
 
-	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{})
+	blah := NewHandshakeManager(tuncidr, preferredRanges, mainHM, &LightHouse{}, &udpConn{}, defaultHandshakeConfig)
 
 	now := time.Now()
 	blah.NextInboundHandshakeTimerTick(now)
@@ -171,12 +171,12 @@ func Test_NewHandshakeManagerIndexcleanup(t *testing.T) {
 	blah.pendingHostMap.AddVpnIPHostInfo(101010, hostinfo)
 	assert.Contains(t, blah.pendingHostMap.Hosts, uint32(101010))
 
-	for i := 1; i <= HandshakeRetries+2; i++ {
-		next_tick := now.Add(HandshakeTryInterval * time.Duration(i))
+	for i := 1; i <= DefaultHandshakeRetries+2; i++ {
+		next_tick := now.Add(DefaultHandshakeTryInterval * time.Duration(i))
 		blah.NextInboundHandshakeTimerTick(next_tick)
 	}
 
-	next_tick := now.Add(HandshakeTryInterval*HandshakeRetries + 3)
+	next_tick := now.Add(DefaultHandshakeTryInterval*DefaultHandshakeRetries + 3)
 	blah.NextInboundHandshakeTimerTick(next_tick)
 	assert.NotContains(t, blah.pendingHostMap.Hosts, uint32(101010))
 	assert.NotContains(t, blah.pendingHostMap.Indexes, uint32(12341234))

main.go

@@ -265,7 +265,13 @@ func Main(configPath string, configTest bool, buildVersion string) {
 		l.WithError(err).Error("Lighthouse unreachable")
 	}
 
-	handshakeManager := NewHandshakeManager(tunCidr, preferredRanges, hostMap, lightHouse, udpServer)
+	handshakeConfig := HandshakeConfig{
+		tryInterval:  config.GetDuration("handshakes.try_interval", DefaultHandshakeTryInterval),
+		retries:      config.GetInt("handshakes.retries", DefaultHandshakeRetries),
+		waitRotation: config.GetInt("handshakes.wait_rotation", DefaultHandshakeWaitRotation),
+	}
+
+	handshakeManager := NewHandshakeManager(tunCidr, preferredRanges, hostMap, lightHouse, udpServer, handshakeConfig)
 
 	//TODO: These will be reused for psk
 	//handshakeMACKey := config.GetString("handshake_mac.key", "")