Tensors for pooling.
| 47 | |
| 48 | @dataclass |
| 49 | class PoolingMetadata: |
| 50 | """Tensors for pooling.""" |
| 51 | |
| 52 | prompt_lens: paddle.Tensor # CPU Tensor |
| 53 | prompt_token_ids: Optional[paddle.Tensor] |
| 54 | pooling_params: list[PoolingParams] |
| 55 | pooling_cursor: Optional[PoolingCursor] = None |
| 56 | |
| 57 | def __getitem__(self, indices: slice): |
| 58 | return PoolingMetadata( |
| 59 | prompt_lens=self.prompt_lens[indices], |
| 60 | prompt_token_ids=None if self.prompt_token_ids is None else self.prompt_token_ids[indices], |
| 61 | pooling_params=self.pooling_params[indices], |
| 62 | pooling_cursor=None if self.pooling_cursor is None else self.pooling_cursor[indices], |
| 63 | ) |
| 64 | |
| 65 | def build_pooling_cursor(self, num_scheduled_tokens: list[int], device: Device): |
| 66 | self.pooling_cursor = build_pooling_cursor(num_scheduled_tokens, self.prompt_lens, device) |
| 67 | |
| 68 | |
| 69 | def build_pooling_cursor(num_scheduled_tokens: list[int], prompt_lens: paddle.Tensor, device: Device): |
no outgoing calls