Function TestDoBatch_QuorumError

ring/ring_test.go:344–507

(t *testing.T)

}

func TestDoBatch_QuorumError(t *testing.T) {
	const (
		// we should run several write request to make sure we don't have any race condition on the batchTracker code
		numberOfOperations = 10000
		replicationFactor  = 3
	)

	gen := initTokenGenerator(t)

	desc := NewDesc()
	for address := 0; address < replicationFactor; address++ {
		instTokens := gen.GenerateTokens(128, nil)
		instanceID := fmt.Sprintf("%d", address)
		desc.AddIngester(instanceID, instanceID, "", instTokens, ACTIVE, time.Now(), false, time.Time{}, nil)
	}
	ringConfig := Config{
		HeartbeatTimeout:  time.Hour,
		ReplicationFactor: replicationFactor,
	}
	ring, err := NewWithStoreClientAndStrategy(ringConfig, "ingester", ringKey, nil, NewDefaultReplicationStrategy(), nil, log.NewNopLogger())
	require.NoError(t, err)
	ring.updateRingState(desc)
	operationKeys := []uint32{1, 10, 100}
	ctx := context.Background()
	unfinishedDoBatchCalls := sync.WaitGroup{}
	runDoBatch := func(instanceReturnErrors [replicationFactor]error, isClientError func(error) bool) error {
		unfinishedDoBatchCalls.Add(1)
		returnInstanceError := func(i InstanceDesc, _ []int) error {
			instanceID, err := strconv.Atoi(i.Addr)
			require.NoError(t, err)
			return instanceReturnErrors[instanceID]
		}
		return DoBatchWithOptions(ctx, Write, ring, operationKeys, returnInstanceError, DoBatchOptions{
			Cleanup:       func() { unfinishedDoBatchCalls.Done() },
			IsClientError: isClientError,
		})
	}

	updateState := func(instanceIDs []string, state InstanceState) {
		for _, instanceID := range instanceIDs {
			inst := ring.ringDesc.Ingesters[instanceID]
			inst.State = state
			inst.Timestamp = time.Now().Unix()
			ring.ringDesc.Ingesters[instanceID] = inst
			ring.updateRingState(ring.ringDesc)
		}
	}

	http429Error := httpgrpc.Errorf(429, "Throttling")
	http500Error := httpgrpc.Errorf(500, "InternalServerError")
	mockClientError := mockError{isClientErr: true}
	mockServerError := mockError{isClientErr: false}

	isClientErr := func(err error) bool {
		if mockErr, ok := err.(mockError); ok {
			return mockErr.isClientError()
		}
		return false