A custom iterator for loading files in batches.
| 83 | |
| 84 | |
| 85 | class Iterator(xgboost.DataIter): |
| 86 | """A custom iterator for loading files in batches.""" |
| 87 | |
| 88 | def __init__( |
| 89 | self, device: Literal["cpu", "cuda"], file_paths: List[Tuple[str, str]] |
| 90 | ) -> None: |
| 91 | self.device = device |
| 92 | |
| 93 | self._file_paths = file_paths |
| 94 | self._it = 0 |
| 95 | # XGBoost will generate some cache files under the current directory with the |
| 96 | # prefix "cache" |
| 97 | super().__init__(cache_prefix=os.path.join(".", "cache")) |
| 98 | |
| 99 | def load_file(self) -> Tuple[np.ndarray, np.ndarray]: |
| 100 | """Load a single batch of data.""" |
| 101 | X_path, y_path = self._file_paths[self._it] |
| 102 | # When the `ExtMemQuantileDMatrix` is used, the device must match. GPU cannot |
| 103 | # consume CPU input data and vice-versa. |
| 104 | if self.device == "cpu": |
| 105 | X = np.load(X_path) |
| 106 | y = np.load(y_path) |
| 107 | else: |
| 108 | import cupy as cp # pylint: disable=import-outside-toplevel |
| 109 | |
| 110 | X = cp.load(X_path) |
| 111 | y = cp.load(y_path) |
| 112 | |
| 113 | assert X.shape[0] == y.shape[0] |
| 114 | return X, y |
| 115 | |
| 116 | def next(self, input_data: Callable) -> bool: |
| 117 | """Advance the iterator by 1 step and pass the data to XGBoost. This function |
| 118 | is called by XGBoost during the construction of ``DMatrix`` |
| 119 | |
| 120 | """ |
| 121 | if self._it == len(self._file_paths): |
| 122 | # return False to let XGBoost know this is the end of iteration |
| 123 | return False |
| 124 | |
| 125 | # input_data is a keyword-only function passed in by XGBoost and has the similar |
| 126 | # signature to the ``DMatrix`` constructor. |
| 127 | X, y = self.load_file() |
| 128 | input_data(data=X, label=y) |
| 129 | self._it += 1 |
| 130 | return True |
| 131 | |
| 132 | def reset(self) -> None: |
| 133 | """Reset the iterator to its beginning""" |
| 134 | self._it = 0 |
| 135 | |
| 136 | |
| 137 | def hist_train(it: Iterator) -> None: |