From 0f6d7949c4194f8a1af822c0a6ad3a45b9d0b249 Mon Sep 17 00:00:00 2001
From: Ingo Oppermann <ingo@datarhei.com>
Date: Tue, 17 Sep 2024 15:08:11 +0200
Subject: [PATCH] Fix deadlock in cluster shutdown

---
 cluster/cluster.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/cluster/cluster.go b/cluster/cluster.go
index 69551277..60e41235 100644
--- a/cluster/cluster.go
+++ b/cluster/cluster.go
@@ -620,7 +620,7 @@ func (c *cluster) CertManager() autocert.Manager {
 }
 
 func (c *cluster) Shutdown() error {
-	c.logger.Info().Log("Shutting down cluster")
+	c.logger.Info().Log("Shutting down cluster ...")
 	c.shutdownLock.Lock()
 	defer c.shutdownLock.Unlock()
 
@@ -631,9 +631,14 @@ func (c *cluster) Shutdown() error {
 	c.shutdown = true
 	close(c.shutdownCh)
 
+	c.logger.Info().Log("Waiting for all routines to exit ...")
+
 	c.shutdownWg.Wait()
 
+	c.logger.Info().Log("All routines exited ...")
+
 	if c.manager != nil {
+		c.logger.Info().Log("Shutting down node manager ...")
 		c.manager.NodesClear()
 	}
 
@@ -641,13 +646,18 @@ func (c *cluster) Shutdown() error {
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
 
+		c.logger.Info().Log("Shutting down API ...")
+
 		c.api.Shutdown(ctx)
 	}
 
 	if c.raft != nil {
+		c.logger.Info().Log("Shutting down raft ...")
 		c.raft.Shutdown()
 	}
 
+	c.logger.Info().Log("Cluster stopped")
+
 	return nil
 }
 
@@ -1029,7 +1039,7 @@ func (c *cluster) trackLeaderChanges() {
 			if !isNodeInCluster {
 				// We're not anymore part of the cluster, shutdown
 				c.logger.Warn().WithField("id", c.nodeID).Log("This node left the cluster. Shutting down.")
-				c.Shutdown()
+				go c.Shutdown()
 			}
 
 		case <-c.shutdownCh: