Method stopping

kv/memberlist/memberlist_client.go:976–1019

While Stopping, we try to leave memberlist cluster and then shutdown memberlist client. We do this in order to send out last messages, typically that ingester has LEFT the ring.

(_ error)

// While Stopping, we try to leave memberlist cluster and then shutdown memberlist client.
// We do this in order to send out last messages, typically that ingester has LEFT the ring.
func (m *KV) stopping(_ error) error {
	// Stop propagation delay tracker if running.
	if m.propagationDelayTracker != nil {
		m.propagationDelayTracker.StopAsync()
		if err := m.propagationDelayTracker.AwaitTerminated(context.Background()); err != nil {
			level.Warn(m.logger).Log("msg", "error stopping propagation delay tracker", "err", err)
		}
	}

	level.Info(m.logger).Log("msg", "leaving memberlist cluster")

	// Wait until queue with locally-generated messages is empty, but don't wait for too long.
	// Also don't wait if there is just one node left.
	// Note: Once we enter Stopping state, we don't queue more locally-generated messages.

	deadline := time.Now().Add(m.cfg.BroadcastTimeoutForLocalUpdatesOnShutdown)

	msgs := m.localBroadcasts.NumQueued()
	nodes := m.memberlist.NumMembers()
	for msgs > 0 && nodes > 1 && (m.cfg.BroadcastTimeoutForLocalUpdatesOnShutdown <= 0 || time.Now().Before(deadline)) {
		level.Info(m.logger).Log("msg", "waiting for locally-generated broadcast messages to be sent out", "count", msgs, "nodes", nodes)
		time.Sleep(250 * time.Millisecond)

		msgs = m.localBroadcasts.NumQueued()
		nodes = m.memberlist.NumMembers()
	}

	if msgs > 0 {
		level.Warn(m.logger).Log("msg", "locally-generated broadcast messages left the queue", "count", msgs, "nodes", nodes)
	}

	err := m.memberlist.Leave(m.cfg.LeaveTimeout)
	if err != nil {
		level.Error(m.logger).Log("msg", "error when leaving memberlist cluster", "err", err)
	}

	close(m.shutdown)

	err = m.memberlist.Shutdown()
	if err != nil {
		level.Error(m.logger).Log("msg", "error when shutting down memberlist client", "err", err)
	}
	return nil
}

// List returns all known keys under a given prefix.
// No communication with other nodes in the cluster is done here.