| 39 | return len(self.data_offsets) - 1 |
| 40 | |
| 41 | class IndexedDatasetBuilder: |
| 42 | def __init__(self, path): |
| 43 | self.path = path |
| 44 | self.out_file = open(f"{path}.data", 'wb') |
| 45 | self.byte_offsets = [0] |
| 46 | |
| 47 | def add_item(self, item): |
| 48 | s = pickle.dumps(item) |
| 49 | bytes = self.out_file.write(s) |
| 50 | self.byte_offsets.append(self.byte_offsets[-1] + bytes) |
| 51 | |
| 52 | def finalize(self): |
| 53 | self.out_file.close() |
| 54 | np.save(open(f"{self.path}.idx", 'wb'), {'offsets': self.byte_offsets}) |
| 55 | |
| 56 | |
| 57 | if __name__ == "__main__": |
no outgoing calls
no test coverage detected