CheckWeightedRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, connected to a server exposing the test.grpc_testing.TestService, are weighted roundrobined (with randomness) across the given backend addresses. Returns a non-nil error if context deadline expires before RPCs start t
(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient, addrs []resolver.Address)
| 140 | // Returns a non-nil error if context deadline expires before RPCs start to get |
| 141 | // roundrobined across the given backends. |
| 142 | func CheckWeightedRoundRobinRPCs(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient, addrs []resolver.Address) error { |
| 143 | if err := waitForTrafficToReachBackends(ctx, client, addrs); err != nil { |
| 144 | return err |
| 145 | } |
| 146 | |
| 147 | // At this point, RPCs are getting successfully executed at the backends |
| 148 | // that we care about. To take the randomness of the WRR into account, we |
| 149 | // look for approximate distribution instead of exact. |
| 150 | wantAddrCount := make(map[string]int) |
| 151 | for _, addr := range addrs { |
| 152 | wantAddrCount[addr.Addr]++ |
| 153 | } |
| 154 | attemptCount := attemptCounts(wantAddrCount) |
| 155 | |
| 156 | expectedCount := make(map[string]float64) |
| 157 | for addr, count := range wantAddrCount { |
| 158 | expectedCount[addr] = float64(count) / float64(len(addrs)) * float64(attemptCount) |
| 159 | } |
| 160 | |
| 161 | // There is a small possibility that RPCs are reaching backends that we |
| 162 | // don't expect them to reach here. The can happen because: |
| 163 | // - at time T0, the list of backends [A, B, C, D]. |
| 164 | // - at time T1, the test updates the list of backends to [A, B, C], and |
| 165 | // immediately starts attempting to check the distribution of RPCs to the |
| 166 | // new backends. |
| 167 | // - there is no way for the test to wait for a new picker to be pushed on |
| 168 | // to the channel (which contains the updated list of backends) before |
| 169 | // starting to attempt the RPC distribution checks. |
| 170 | // - This is usually a transitory state and will eventually fix itself when |
| 171 | // the new picker is pushed on the channel, and RPCs will start getting |
| 172 | // routed to only backends that we care about. |
| 173 | // |
| 174 | // We work around this situation by using two loops. The inner loop contains |
| 175 | // the meat of the calculations, and includes the logic which factors out |
| 176 | // the randomness in weighted roundrobin. If we ever see an RPCs getting |
| 177 | // routed to a backend that we don't expect it to get routed to, we break |
| 178 | // from the inner loop thereby resetting all state and start afresh. |
| 179 | for { |
| 180 | observedCount := make(map[string]float64) |
| 181 | InnerLoop: |
| 182 | for { |
| 183 | if ctx.Err() != nil { |
| 184 | return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) |
| 185 | } |
| 186 | for i := 0; i < attemptCount; i++ { |
| 187 | var peer peer.Peer |
| 188 | if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { |
| 189 | return fmt.Errorf("EmptyCall() = %v, want <nil>", err) |
| 190 | } |
| 191 | if addr := peer.Addr.String(); wantAddrCount[addr] == 0 { |
| 192 | break InnerLoop |
| 193 | } |
| 194 | observedCount[peer.Addr.String()]++ |
| 195 | } |
| 196 | |
| 197 | return pearsonsChiSquareTest(t, observedCount, expectedCount) |
| 198 | } |
| 199 | <-time.After(time.Millisecond) |