Merge pull request #1984 from gravitl/bugfix_node_expiration

Add expired nodes to zombie list and send peer update on deletion

logic/zombie.go

@@ -25,6 +25,7 @@ var (
 
 // CheckZombies - checks if new node has same macaddress as existing node
 // if so, existing node is added to zombie node quarantine list
+// also cleans up nodes past their expiration date
 func CheckZombies(newnode *models.Node, mac net.HardwareAddr) {
 	nodes, err := GetNetworkNodes(newnode.Network)
 	if err != nil {
@@ -37,7 +38,7 @@ func CheckZombies(newnode *models.Node, mac net.HardwareAddr) {
 			// should we delete the node if host not found ??
 			continue
 		}
-		if host.MacAddress.String() == mac.String() {
+		if host.MacAddress.String() == mac.String() || time.Now().After(node.ExpirationDateTime) {
 			logger.Log(0, "adding ", node.ID.String(), " to zombie list")
 			newZombie <- node.ID
 		}
@@ -45,7 +46,7 @@ func CheckZombies(newnode *models.Node, mac net.HardwareAddr) {
 }
 
 // ManageZombies - goroutine which adds/removes/deletes nodes from the zombie node quarantine list
-func ManageZombies(ctx context.Context) {
+func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) {
 	logger.Log(2, "Zombie management started")
 	InitializeZombies()
 	for {
@@ -80,11 +81,13 @@ func ManageZombies(ctx context.Context) {
 						zombies = append(zombies[:i], zombies[i+1:]...)
 						continue
 					}
-					if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME {
+					if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME || time.Now().After(node.ExpirationDateTime) {
 						if err := DeleteNode(&node, true); err != nil {
 							logger.Log(1, "error deleting zombie node", zombies[i].String(), err.Error())
 							continue
 						}
+						node.Action = models.NODE_DELETE
+						peerUpdate <- &node
 						logger.Log(1, "deleting zombie node", node.ID.String())
 						zombies = append(zombies[:i], zombies[i+1:]...)
 					}

main.go

@@ -191,7 +191,15 @@ func runMessageQueue(wg *sync.WaitGroup) {
 	mq.SetupMQTT()
 	ctx, cancel := context.WithCancel(context.Background())
 	go mq.Keepalive(ctx)
-	go logic.ManageZombies(ctx)
+	go func() {
+		peerUpdate := make(chan *models.Node)
+		go logic.ManageZombies(ctx, peerUpdate)
+		for nodeUpdate := range peerUpdate {
+			if err := mq.NodeUpdate(nodeUpdate); err != nil {
+				logger.Log(0, "failed to send peer update for deleted node: ", nodeUpdate.ID.String(), err.Error())
+			}
+		}
+	}()
 	go logic.PurgePendingNodes(ctx)
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, syscall.SIGTERM, os.Interrupt)