Function TestRing_ShuffleShard_ConsistencyOnZonesChanged

ring/ring_test.go:2313–2377

(t *testing.T)

}

func TestRing_ShuffleShard_ConsistencyOnZonesChanged(t *testing.T) {
	// Create 20 instances in 2 zones.
	ringInstances := map[string]InstanceDesc{}
	for i := 0; i < 20; i++ {
		name, desc, _ := generateRingInstance(initTokenGenerator(t), i, i%2, 128, nil)
		ringInstances[name] = desc
	}

	// Init the ring.
	ringDesc := &Desc{Ingesters: ringInstances}
	ring := newRingForTesting(Config{
		HeartbeatTimeout:     time.Hour,
		ZoneAwarenessEnabled: true,
	}, false)
	ring.setRingStateFromDesc(ringDesc, false, false, false)

	// Get the replication set with shard size = 2.
	firstShard := ring.ShuffleShard("tenant-id", 2)
	assert.Equal(t, 2, firstShard.InstancesCount())

	firstSet, err := firstShard.GetAllHealthy(Read)
	require.NoError(t, err)

	// Increase shard size to 4.
	secondShard := ring.ShuffleShard("tenant-id", 4)
	assert.Equal(t, 4, secondShard.InstancesCount())

	secondSet, err := secondShard.GetAllHealthy(Read)
	require.NoError(t, err)

	for _, firstInstance := range firstSet.Instances {
		assert.True(t, secondSet.Includes(firstInstance.Addr), "new replication set is expected to include previous instance %s", firstInstance.Addr)
	}

	// Scale up cluster, adding 10 instances in 1 new zone.
	for i := 20; i < 30; i++ {
		name, desc, _ := generateRingInstance(initTokenGenerator(t), i, 2, 128, nil)
		ringInstances[name] = desc
	}

	ringDesc.Ingesters = ringInstances
	ring.setRingStateFromDesc(ringDesc, false, false, false)

	// Increase shard size to 6.
	thirdShard := ring.ShuffleShard("tenant-id", 6)
	assert.Equal(t, 6, thirdShard.InstancesCount())

	thirdSet, err := thirdShard.GetAllHealthy(Read)
	require.NoError(t, err)

	for _, secondInstance := range secondSet.Instances {
		assert.True(t, thirdSet.Includes(secondInstance.Addr), "new replication set is expected to include previous instance %s", secondInstance.Addr)
	}

	// Increase shard size to 9.
	fourthShard := ring.ShuffleShard("tenant-id", 9)
	assert.Equal(t, 9, fourthShard.InstancesCount())