| 96 | } |
| 97 | |
| 98 | func sloHook(allByTenantCounter, withinSLOByTenantCounter *prometheus.CounterVec, throughputVec *prometheus.CounterVec, inspectedBytesVec *prometheus.CounterVec, cfg SLOConfig) handlerPostHook { |
| 99 | return func(resp *http.Response, tenant string, bytesProcessed uint64, latency time.Duration, err error) { |
| 100 | // most errors are SLO violations but we have few exceptions. |
| 101 | if err != nil { |
| 102 | // However, gRPC resource exhausted error (429), invalid argument (400), not found (404) and |
| 103 | // request cancellations are considered within the SLO. |
| 104 | switch status.Code(err) { |
| 105 | case codes.ResourceExhausted, codes.InvalidArgument, codes.NotFound: |
| 106 | allByTenantCounter.WithLabelValues(tenant, resultCompleted).Inc() |
| 107 | withinSLOByTenantCounter.WithLabelValues(tenant, resultCompleted).Inc() |
| 108 | return |
| 109 | } |
| 110 | |
| 111 | if grpcutil.IsCanceled(err) { |
| 112 | allByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 113 | withinSLOByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 114 | return |
| 115 | } |
| 116 | |
| 117 | // check for the response and 499 in the status code, can come from http pipeline along with error |
| 118 | if resp != nil && resp.StatusCode == util.StatusClientClosedRequest { |
| 119 | allByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 120 | withinSLOByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 121 | return |
| 122 | } |
| 123 | |
| 124 | // in case we have error, that doesn't fall into the above categories, it's a SLO violation |
| 125 | // so only increment the allByTenantCounter |
| 126 | allByTenantCounter.WithLabelValues(tenant, resultCompleted).Inc() |
| 127 | return |
| 128 | } |
| 129 | |
| 130 | // we don't always get error in case of http pipeline, check for 499 status code |
| 131 | if resp != nil && resp.StatusCode == util.StatusClientClosedRequest { |
| 132 | allByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 133 | withinSLOByTenantCounter.WithLabelValues(tenant, resultCanceled).Inc() |
| 134 | return |
| 135 | } |
| 136 | |
| 137 | // record all queries |
| 138 | allByTenantCounter.WithLabelValues(tenant, resultCompleted).Inc() |
| 139 | |
| 140 | // all 200s/300s/400s are success |
| 141 | if resp != nil && resp.StatusCode >= 500 { |
| 142 | return |
| 143 | } |
| 144 | |
| 145 | inspectedBytesVec.WithLabelValues(tenant).Add(float64(bytesProcessed)) |
| 146 | |
| 147 | passedThroughput := false |
| 148 | // final check is throughput |
| 149 | // throughputVec is nil for TraceByIDSLO |
| 150 | if cfg.ThroughputBytesSLO > 0 && throughputVec != nil { |
| 151 | throughput := 0.0 |
| 152 | seconds := latency.Seconds() |
| 153 | if seconds > 0 { |
| 154 | throughput = float64(bytesProcessed) / seconds |
| 155 | } |