queryMetrics performs a TraceQL metrics query and verifies the results. It is a basic smoke test to ensure that the traceql query is working. It randomly selects an attribute from an expected trace and queries for its presence in the metrics API within a time window around the seed time.
(client httpclient.TempoHTTPClient, seed time.Time, config vultureConfiguration, l *zap.Logger)
| 660 | // It randomly selects an attribute from an expected trace and queries for its presence |
| 661 | // in the metrics API within a time window around the seed time. |
| 662 | func queryMetrics(client httpclient.TempoHTTPClient, seed time.Time, config vultureConfiguration, l *zap.Logger) (traceMetrics, error) { |
| 663 | tm := traceMetrics{ |
| 664 | requested: 1, |
| 665 | } |
| 666 | |
| 667 | info := util.NewTraceInfo(seed, config.tempoOrgID) |
| 668 | hexID := info.HexID() |
| 669 | |
| 670 | expected, err := info.ConstructTraceFromEpoch() |
| 671 | if err != nil { |
| 672 | err = fmt.Errorf("unable to construct trace from epoch: %w", err) |
| 673 | return traceMetrics{}, err |
| 674 | } |
| 675 | |
| 676 | attr := util.RandomAttrFromTrace(expected) |
| 677 | if attr == nil { |
| 678 | tm.notFoundSearchAttribute++ |
| 679 | return tm, fmt.Errorf("no search attr selected from trace") |
| 680 | } |
| 681 | |
| 682 | logger := l.With( |
| 683 | zap.Int64("seed", seed.Unix()), |
| 684 | zap.String("hexID", hexID), |
| 685 | zap.Duration("ago", time.Since(seed)), |
| 686 | zap.String("key", attr.Key), |
| 687 | zap.String("value", util.StringifyAnyValue(attr.Value)), |
| 688 | ) |
| 689 | logger.Info("searching Tempo via metrics") |
| 690 | |
| 691 | // Use the API to find details about the expected trace. give an hour range around the seed. |
| 692 | start := seed.Add(-30 * time.Minute).Unix() |
| 693 | end := seed.Add(30 * time.Minute).Unix() |
| 694 | |
| 695 | resp, err := client.MetricsQueryRange( |
| 696 | fmt.Sprintf(`{.%s = "%s"} | count_over_time()`, attr.Key, util.StringifyAnyValue(attr.Value)), |
| 697 | start, end, "10s", 0, |
| 698 | ) |
| 699 | if err != nil { |
| 700 | logger.Error("failed to query metrics", zap.Error(err)) |
| 701 | tm.requestFailed++ |
| 702 | return tm, err |
| 703 | } |
| 704 | |
| 705 | if len(resp.Series) == 0 { |
| 706 | tm.notFoundByMetrics++ |
| 707 | logger.Error("failed to find trace by metrics", zap.Error(err)) |
| 708 | return tm, fmt.Errorf("expected trace %s not found in metrics", hexID) |
| 709 | } |
| 710 | |
| 711 | if len(resp.Series) > 1 { |
| 712 | tm.incorrectMetricsResult++ |
| 713 | return tm, fmt.Errorf("expected exactly 1 series, got %d", len(resp.Series)) |
| 714 | } |
| 715 | timeSeries := resp.Series[0] |
| 716 | if timeSeries == nil { |
| 717 | tm.incorrectMetricsResult++ |
| 718 | return tm, errors.New("expected time series, got nil") |
| 719 | } |