Wrapper for Ray Native operations.
| 209 | |
| 210 | |
| 211 | class StandardRayWrapper: |
| 212 | """Wrapper for Ray Native operations.""" |
| 213 | |
| 214 | def read_parquet(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 215 | """Read parquet files using standard Ray.""" |
| 216 | return ray.data.read_parquet(path, **kwargs) |
| 217 | |
| 218 | def read_csv(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 219 | """Read CSV files using standard Ray.""" |
| 220 | return ray.data.read_csv(path, **kwargs) |
| 221 | |
| 222 | def read_json(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 223 | """Read JSON/JSONL file(s).""" |
| 224 | return ray.data.read_json(path, **kwargs) |
| 225 | |
| 226 | def read_text(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 227 | """Read plain-text file(s).""" |
| 228 | return ray.data.read_text(path, **kwargs) |
| 229 | |
| 230 | def read_images(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 231 | """Read image files (PNG, JPEG, …) from a directory as numpy arrays.""" |
| 232 | return ray.data.read_images(path, **kwargs) |
| 233 | |
| 234 | def read_binary_files(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 235 | """Read arbitrary binary files; each row has 'path' and 'bytes' columns.""" |
| 236 | return ray.data.read_binary_files(path, **kwargs) |
| 237 | |
| 238 | def read_tfrecords(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 239 | """Read TFRecord file(s).""" |
| 240 | return ray.data.read_tfrecords(path, **kwargs) |
| 241 | |
| 242 | def read_webdataset(self, path: Union[str, List[str]], **kwargs) -> Any: |
| 243 | """Read WebDataset tar shard(s).""" |
| 244 | return ray.data.read_webdataset(path, **kwargs) |
| 245 | |
| 246 | def read_mongo(self, uri: str, database: str, collection: str, **kwargs) -> Any: |
| 247 | """Read from a MongoDB collection.""" |
| 248 | return ray.data.read_mongo( |
| 249 | uri=uri, database=database, collection=collection, **kwargs |
| 250 | ) |
| 251 | |
| 252 | def read_sql(self, sql: str, connection_url: str, **kwargs) -> Any: |
| 253 | """Read from a SQL database. Builds the connection factory from connection_url.""" |
| 254 | import sqlalchemy |
| 255 | |
| 256 | def _connection_factory(): |
| 257 | # raw_connection() returns a DB API2-compliant connection (with .cursor()) |
| 258 | # from the underlying driver. engine.connect() returns a SQLAlchemy 2.0 |
| 259 | # Connection object which does NOT expose .cursor(), failing Ray Data's |
| 260 | # DB API2 compliance check. |
| 261 | return sqlalchemy.create_engine(connection_url).raw_connection() |
| 262 | |
| 263 | return ray.data.read_sql(sql, _connection_factory, **kwargs) |
| 264 | |
| 265 | def from_huggingface( |
| 266 | self, dataset_name: str, split: str = "train", **kwargs |
| 267 | ) -> Any: |
| 268 | """Load a HuggingFace dataset and convert to a Ray Dataset.""" |