Use dummy inputs to run before formal execution. Args: num_tokens: expected_decode_len: Expected number of tokens generated in_capturing: Is cuda graph in capturing state capture_prefill: Capture pure prefill for cuda graph
(
self,
num_tokens: int,
batch_size: int,
expected_decode_len: int = 1,
in_capturing: bool = False,
capture_prefill: bool = False,
accept_all_drafts: bool = False,
reject_all_drafts: bool = False,
)
| 1921 | return sampler_output |
| 1922 | |
| 1923 | def _dummy_run( |
| 1924 | self, |
| 1925 | num_tokens: int, |
| 1926 | batch_size: int, |
| 1927 | expected_decode_len: int = 1, |
| 1928 | in_capturing: bool = False, |
| 1929 | capture_prefill: bool = False, |
| 1930 | accept_all_drafts: bool = False, |
| 1931 | reject_all_drafts: bool = False, |
| 1932 | ) -> paddle.Tensor: |
| 1933 | """ |
| 1934 | Use dummy inputs to run before formal execution. |
| 1935 | Args: |
| 1936 | num_tokens: |
| 1937 | expected_decode_len: Expected number of tokens generated |
| 1938 | in_capturing: Is cuda graph in capturing state |
| 1939 | capture_prefill: Capture pure prefill for cuda graph |
| 1940 | accept_all_drafts: Target model will accept all draft tokens |
| 1941 | reject_all_drafts: Target model will reject all draft tokens |
| 1942 | """ |
| 1943 | input_length_list, max_dec_len_list, block_num = self.get_input_length_list( |
| 1944 | num_tokens=num_tokens, |
| 1945 | batch_size=batch_size, |
| 1946 | expected_decode_len=expected_decode_len, |
| 1947 | capture_prefill=capture_prefill, |
| 1948 | ) |
| 1949 | self._dummy_prefill_inputs( |
| 1950 | input_length_list=input_length_list, |
| 1951 | max_dec_len_list=max_dec_len_list, |
| 1952 | block_num=block_num, |
| 1953 | ) |
| 1954 | if self.speculative_method in ["mtp"]: |
| 1955 | self.proposer.dummy_prefill_inputs( |
| 1956 | num_tokens=num_tokens, |
| 1957 | batch_size=batch_size, |
| 1958 | expected_decode_len=expected_decode_len, |
| 1959 | ) |
| 1960 | |
| 1961 | while True: |
| 1962 | # 1. Initialize forward meta and attention meta data |
| 1963 | self._prepare_inputs(is_dummy_or_profile_run=True) |
| 1964 | # 2. Padding inputs for cuda graph |
| 1965 | self.forward_meta.step_use_cudagraph = in_capturing and self.forward_meta.step_use_cudagraph |
| 1966 | self.padding_cudagraph_inputs() |
| 1967 | |
| 1968 | # 3. Run model |
| 1969 | if self.enable_mm: |
| 1970 | model_output = self.model( |
| 1971 | self.forward_meta.ids_remove_padding, |
| 1972 | self.share_inputs["image_features"], |
| 1973 | self.forward_meta, |
| 1974 | ) |
| 1975 | else: |
| 1976 | # fallback paddleformers use cuda graph need kwargs |
| 1977 | model_output = self.model( |
| 1978 | ids_remove_padding=self.forward_meta.ids_remove_padding, |
| 1979 | forward_meta=self.forward_meta, |
| 1980 | ) |
no test coverage detected