| 207 | } |
| 208 | |
| 209 | func (g *Generator) getOrCreateInstance(instanceID string) (*instance, error) { |
| 210 | // Fast path: check with read lock first |
| 211 | inst, ok := g.getInstanceByID(instanceID) |
| 212 | if ok { |
| 213 | return inst, nil |
| 214 | } |
| 215 | |
| 216 | g.instancesMtx.Lock() |
| 217 | defer g.instancesMtx.Unlock() |
| 218 | |
| 219 | // Double-check after acquiring write lock |
| 220 | if inst, ok := g.instances[instanceID]; ok { |
| 221 | return inst, nil |
| 222 | } |
| 223 | |
| 224 | // Check if this instance creation failed previously |
| 225 | if failedAt, ok := g.failedInstances[instanceID]; ok { |
| 226 | if time.Since(failedAt) < failureBackoff { |
| 227 | return nil, errInstanceCreationBackoff |
| 228 | } |
| 229 | // Backoff expired, clear the failure and retry |
| 230 | delete(g.failedInstances, instanceID) |
| 231 | } |
| 232 | |
| 233 | inst, err := g.createInstance(instanceID) |
| 234 | if err != nil { |
| 235 | g.failedInstances[instanceID] = time.Now() |
| 236 | level.Error(g.logger).Log("msg", "instance creation failed, will retry after backoff", |
| 237 | "backoff", failureBackoff, "tenant", instanceID, "err", err) |
| 238 | return nil, err |
| 239 | } |
| 240 | |
| 241 | g.instances[instanceID] = inst |
| 242 | return inst, nil |
| 243 | } |
| 244 | |
| 245 | func (g *Generator) getInstanceByID(id string) (*instance, bool) { |
| 246 | g.instancesMtx.RLock() |