()
| 721 | } |
| 722 | |
| 723 | func waitForSentinelClusterStable() { |
| 724 | sentinel1 := redis.NewSentinelClient(&redis.Options{ |
| 725 | Addr: ":" + sentinelPort1, |
| 726 | }) |
| 727 | defer sentinel1.Close() |
| 728 | |
| 729 | sentinel2 := redis.NewSentinelClient(&redis.Options{ |
| 730 | Addr: ":" + sentinelPort2, |
| 731 | }) |
| 732 | defer sentinel2.Close() |
| 733 | |
| 734 | sentinel3 := redis.NewSentinelClient(&redis.Options{ |
| 735 | Addr: ":" + sentinelPort3, |
| 736 | }) |
| 737 | defer sentinel3.Close() |
| 738 | |
| 739 | Eventually(func() bool { |
| 740 | masterInfo1, err1 := sentinel1.Master(ctx, sentinelName).Result() |
| 741 | masterInfo2, err2 := sentinel2.Master(ctx, sentinelName).Result() |
| 742 | masterInfo3, err3 := sentinel3.Master(ctx, sentinelName).Result() |
| 743 | |
| 744 | if err1 != nil || err2 != nil || err3 != nil { |
| 745 | return false |
| 746 | } |
| 747 | // Check master ip and port are consistent across all sentinels |
| 748 | if masterInfo1["ip"] != masterInfo2["ip"] || |
| 749 | masterInfo1["port"] != masterInfo2["port"] || |
| 750 | masterInfo2["ip"] != masterInfo3["ip"] || |
| 751 | masterInfo2["port"] != masterInfo3["port"] { |
| 752 | return false |
| 753 | } |
| 754 | |
| 755 | // Each sentinel must report at least 2 usable (non-down, |
| 756 | // non-disconnected) replicas. Just counting replicas isn't enough: |
| 757 | // production code filters disconnected replicas out, so a sentinel |
| 758 | // reporting "2 replicas, both disconnected" yields 0 usable nodes |
| 759 | // and triggers the silent master fallback. |
| 760 | replicas1, err1 := sentinel1.Replicas(ctx, sentinelName).Result() |
| 761 | replicas2, err2 := sentinel2.Replicas(ctx, sentinelName).Result() |
| 762 | replicas3, err3 := sentinel3.Replicas(ctx, sentinelName).Result() |
| 763 | |
| 764 | if err1 != nil || err2 != nil || err3 != nil { |
| 765 | return false |
| 766 | } |
| 767 | u1, u2, u3 := countUsableReplicas(replicas1), countUsableReplicas(replicas2), countUsableReplicas(replicas3) |
| 768 | if u1 < 2 || u2 < 2 || u3 < 2 { |
| 769 | return false |
| 770 | } |
| 771 | return u1 == u2 && u2 == u3 |
| 772 | }, "30s", "1s").Should(BeTrue()) |
| 773 | |
| 774 | // End-to-end probe: open the same kind of client the post-failover |
| 775 | // ReadOnly spec uses and confirm it actually routes to a replica. |
| 776 | // This is the deterministic equivalent of the previous 10-second |
| 777 | // time.Sleep — succeeds as soon as the precondition is met, instead of |
| 778 | // hoping 10 seconds is long enough for replicas to come out of the |
| 779 | // "disconnected" state on every sentinel. |
| 780 | Eventually(func() bool { |
no test coverage detected