waitForVFs waits for Virtual Functions to be created on all NVIDIA GPUs. It polls sriov_numvfs until all GPUs have their full VF count enabled.
(ctx context.Context, timeout time.Duration)
| 1768 | // waitForVFs waits for Virtual Functions to be created on all NVIDIA GPUs. |
| 1769 | // It polls sriov_numvfs until all GPUs have their full VF count enabled. |
| 1770 | func waitForVFs(ctx context.Context, timeout time.Duration) error { |
| 1771 | pollInterval := time.Duration(sleepIntervalSecondsFlag) * time.Second |
| 1772 | nvpciLib := nvpci.New() |
| 1773 | |
| 1774 | return wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, func(ctx context.Context) (bool, error) { |
| 1775 | gpus, err := nvpciLib.GetGPUs() |
| 1776 | if err != nil { |
| 1777 | log.Warnf("Error getting GPUs: %v", err) |
| 1778 | return false, nil |
| 1779 | } |
| 1780 | |
| 1781 | var totalExpected, totalEnabled uint64 |
| 1782 | var pfCount int |
| 1783 | for _, gpu := range gpus { |
| 1784 | sriovInfo := gpu.SriovInfo |
| 1785 | if sriovInfo.IsPF() { |
| 1786 | pfCount++ |
| 1787 | totalExpected += sriovInfo.PhysicalFunction.TotalVFs |
| 1788 | totalEnabled += sriovInfo.PhysicalFunction.NumVFs |
| 1789 | } |
| 1790 | } |
| 1791 | |
| 1792 | if totalExpected == 0 { |
| 1793 | log.Info("No SR-IOV capable GPUs found, skipping VF wait") |
| 1794 | return true, nil |
| 1795 | } |
| 1796 | |
| 1797 | if totalEnabled == totalExpected { |
| 1798 | log.Infof("All %d VF(s) enabled on %d NVIDIA GPU(s)", totalEnabled, pfCount) |
| 1799 | return true, nil |
| 1800 | } |
| 1801 | |
| 1802 | log.Infof("Waiting for VFs: %d/%d enabled across %d GPU(s)", totalEnabled, totalExpected, pfCount) |
| 1803 | return false, nil |
| 1804 | }) |
| 1805 | } |
| 1806 | |
| 1807 | func (c *CCManager) validate() error { |
| 1808 | // delete status file if already present |