| 693 | } |
| 694 | |
| 695 | func (i *instance) deleteOldBlocks() error { |
| 696 | i.blocksMtx.Lock() |
| 697 | defer i.blocksMtx.Unlock() |
| 698 | |
| 699 | cutoff := time.Now().Add(-i.Cfg.CompleteBlockTimeout) // Delete blocks older than Complete Block Timeout |
| 700 | |
| 701 | snap := i.blocks.Load() |
| 702 | newSnap := snap |
| 703 | defer func() { |
| 704 | if newSnap != snap { |
| 705 | i.blocks.Store(newSnap) |
| 706 | } |
| 707 | }() |
| 708 | |
| 709 | for id, walBlock := range snap.walBlocks { |
| 710 | if walBlock.BlockMeta().EndTime.Before(cutoff) { |
| 711 | if _, ok := snap.completeBlocks[id]; !ok { |
| 712 | level.Warn(i.logger).Log("msg", "deleting WAL block that was never completed", "block", id.String()) |
| 713 | } |
| 714 | if err := walBlock.Tombstone(); err != nil { |
| 715 | return err |
| 716 | } |
| 717 | newSnap = newSnap.withWALBlockRemoved(id) |
| 718 | wb := walBlock |
| 719 | i.reclaim.add(id, i.tenantID, "wal", func() error { return wb.Clear() }) |
| 720 | } |
| 721 | } |
| 722 | |
| 723 | for id, completeBlock := range snap.completeBlocks { |
| 724 | if !i.completeBlockLifecycle.shouldDeleteCompleteBlock(completeBlock, cutoff) { |
| 725 | continue |
| 726 | } |
| 727 | |
| 728 | level.Info(i.logger).Log("msg", "deleting complete block", "block", id.String()) |
| 729 | if err := i.wal.LocalBackend().TombstoneBlock(id, i.tenantID); err != nil { |
| 730 | return err |
| 731 | } |
| 732 | newSnap = newSnap.withCompleteBlockRemoved(id) |
| 733 | bid := id |
| 734 | tenant := i.tenantID |
| 735 | bk := i.wal.LocalBackend() |
| 736 | i.reclaim.add(bid, tenant, "complete", func() error { return bk.ClearBlock(bid, tenant) }) |
| 737 | } |
| 738 | |
| 739 | return nil |
| 740 | } |