(t *testing.T)
| 37 | ) |
| 38 | |
| 39 | func TestIngestionLimits(t *testing.T) { |
| 40 | util.RunIntegrationTests(t, util.TestHarnessConfig{ |
| 41 | ConfigOverlay: configIngest, |
| 42 | }, func(h *util.TempoHarness) { |
| 43 | h.WaitTracesWritable(t) |
| 44 | |
| 45 | // should fail b/c the trace is too large. each batch should be ~70 bytes |
| 46 | batch := util.MakeThriftBatchWithSpanCount(2) |
| 47 | require.NoError(t, h.WriteJaegerBatch(batch, "")) |
| 48 | |
| 49 | // push a trace |
| 50 | require.NoError(t, h.WriteJaegerBatch(util.MakeThriftBatchWithSpanCount(1), "")) |
| 51 | |
| 52 | // should fail b/c this will be too many traces |
| 53 | batch = util.MakeThriftBatch() |
| 54 | require.NoError(t, h.WriteJaegerBatch(batch, "")) |
| 55 | |
| 56 | // should fail b/c due to ingestion rate limit |
| 57 | batch = util.MakeThriftBatchWithSpanCount(10) |
| 58 | err := h.WriteJaegerBatch(batch, "") |
| 59 | |
| 60 | // this error must have a retryinfo as expected in otel collector code: https://github.com/open-telemetry/opentelemetry-collector/blob/d7b49df5d9e922df6ce56ad4b64ee1c79f9dbdbe/exporter/otlpexporter/otlp.go#L172 |
| 61 | st, ok := status.FromError(err) |
| 62 | require.True(t, ok) |
| 63 | foundRetryInfo := false |
| 64 | for _, detail := range st.Details() { |
| 65 | if _, ok := detail.(*errdetails.RetryInfo); ok { |
| 66 | foundRetryInfo = true |
| 67 | break |
| 68 | } |
| 69 | } |
| 70 | require.True(t, foundRetryInfo) |
| 71 | |
| 72 | // test limit metrics |
| 73 | liveStore := h.Services[util.ServiceLiveStoreZoneA] |
| 74 | err = liveStore.WaitSumMetricsWithOptions(e2e.Equals(2), |
| 75 | []string{"tempo_discarded_spans_total"}, |
| 76 | e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "trace_too_large")), |
| 77 | e2e.WaitMissingMetrics, |
| 78 | ) |
| 79 | require.NoError(t, err) |
| 80 | err = liveStore.WaitSumMetricsWithOptions(e2e.Equals(1), |
| 81 | []string{"tempo_discarded_spans_total"}, |
| 82 | e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "live_traces_exceeded")), |
| 83 | e2e.WaitMissingMetrics, |
| 84 | ) |
| 85 | require.NoError(t, err) |
| 86 | err = h.Services[util.ServiceDistributor].WaitSumMetricsWithOptions(e2e.Equals(10), |
| 87 | []string{"tempo_discarded_spans_total"}, |
| 88 | e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "reason", "rate_limited")), |
| 89 | e2e.WaitMissingMetrics, |
| 90 | ) |
| 91 | require.NoError(t, err) |
| 92 | }) |
| 93 | } |
| 94 | |
| 95 | func TestOTLPLimits(t *testing.T) { |
| 96 | util.RunIntegrationTests(t, util.TestHarnessConfig{ |
nothing calls this directly
no test coverage detected