Method _pool

fastdeploy/worker/gpu_model_runner.py:2648–2693

(self, hidden_states: paddle.Tensor, num_running_requests: int)

        )

    def _pool(self, hidden_states: paddle.Tensor, num_running_requests: int) -> Optional[ModelRunnerOutput]:
        num_scheduled_tokens = int(self.share_inputs["seq_lens_this_time"][:num_running_requests].sum())
        hidden_states = hidden_states[:num_scheduled_tokens]

        prompt_lens = self.share_inputs["prompt_lens"][:num_running_requests]
        prompt_token_ids = self.share_inputs["prompt_ids"]

        pooling_metadata = PoolingMetadata(
            prompt_lens=prompt_lens,
            prompt_token_ids=prompt_token_ids,
            pooling_params=self.pooling_params,
        )

        num_scheduled_tokens_list = [
            int(self.share_inputs["seq_lens_this_time"][i]) for i in range(num_running_requests)
        ]

        device_str = "gpu" if hidden_states.place.is_gpu_place() else "cpu"
        pooling_metadata.build_pooling_cursor(num_scheduled_tokens_list, device=device_str)

        raw_pooler_output = self.model.pooler(hidden_states=hidden_states, pooling_metadata=pooling_metadata)

        seq_lens_decoder = self.share_inputs["seq_lens_decoder"][:num_running_requests]
        seq_lens_encoder = self.share_inputs["seq_lens_encoder"][:num_running_requests]

        pooler_output: list[Optional[paddle.Tensor]] = []
        pooler_output_idx = 0

        for i, prompt_len in enumerate(pooling_metadata.prompt_lens):
            current_seq_len = num_scheduled_tokens_list[i]

            if current_seq_len == 0:
                pooler_output.append(None)
                continue

            total_processed = int(seq_lens_decoder[i]) + int(seq_lens_encoder[i])

            if total_processed == int(prompt_len):
                output = raw_pooler_output[pooler_output_idx]
            else:
                output = None

            pooler_output.append(output)
            pooler_output_idx += 1

        return PoolerOutput(outputs=pooler_output)

    def _execute_empty_input(self, forward_meta) -> None:
        """