| 180 | } |
| 181 | |
| 182 | func (d *Detector) run(t time.Time) Stats { |
| 183 | ctx, cancel := context.WithTimeout(d.ctx, 5*time.Minute) |
| 184 | defer cancel() |
| 185 | |
| 186 | stats := Stats{ |
| 187 | TerminatedJobIDs: []uuid.UUID{}, |
| 188 | Error: nil, |
| 189 | } |
| 190 | |
| 191 | // Find all provisioner jobs to be reaped |
| 192 | jobs, err := d.db.GetProvisionerJobsToBeReaped(ctx, database.GetProvisionerJobsToBeReapedParams{ |
| 193 | PendingSince: t.Add(-PendingJobDuration), |
| 194 | HungSince: t.Add(-HungJobDuration), |
| 195 | MaxJobs: MaxJobsPerRun, |
| 196 | }) |
| 197 | if err != nil { |
| 198 | stats.Error = xerrors.Errorf("get provisioner jobs to be reaped: %w", err) |
| 199 | return stats |
| 200 | } |
| 201 | |
| 202 | jobsToReap := make([]*jobToReap, 0, len(jobs)) |
| 203 | |
| 204 | for _, job := range jobs { |
| 205 | j := &jobToReap{ |
| 206 | ID: job.ID, |
| 207 | } |
| 208 | if job.JobStatus == database.ProvisionerJobStatusPending { |
| 209 | j.Threshold = PendingJobDuration |
| 210 | j.Type = Pending |
| 211 | } else { |
| 212 | j.Threshold = HungJobDuration |
| 213 | j.Type = Hung |
| 214 | } |
| 215 | jobsToReap = append(jobsToReap, j) |
| 216 | } |
| 217 | |
| 218 | // Send a message into the build log for each hung or pending job saying that it |
| 219 | // has been detected and will be terminated, then mark the job as failed. |
| 220 | for _, job := range jobsToReap { |
| 221 | log := d.log.With(slog.F("job_id", job.ID)) |
| 222 | |
| 223 | err := reapJob(ctx, log, d.db, d.pubsub, job) |
| 224 | if err != nil { |
| 225 | if !(xerrors.As(err, &acquireLockError{}) || xerrors.As(err, &jobIneligibleError{})) { |
| 226 | log.Error(ctx, "error forcefully terminating provisioner job", slog.F("type", job.Type), slog.Error(err)) |
| 227 | } |
| 228 | continue |
| 229 | } |
| 230 | |
| 231 | stats.TerminatedJobIDs = append(stats.TerminatedJobIDs, job.ID) |
| 232 | } |
| 233 | |
| 234 | return stats |
| 235 | } |
| 236 | |
| 237 | func reapJob(ctx context.Context, log slog.Logger, db database.Store, pub pubsub.Pubsub, jobToReap *jobToReap) error { |
| 238 | var lowestLogID int64 |