(
self,
hidden_states,
attention_mask=None,
head_mask=None,
encoder_hidden_states=None,
encoder_attention_mask=None,
past_key_values=None,
use_cache=None,
output_attentions=False,
output_hidden_states=False,
return_dict=True,
)
| 476 | self.gradient_checkpointing = False |
| 477 | |
| 478 | def forward( |
| 479 | self, |
| 480 | hidden_states, |
| 481 | attention_mask=None, |
| 482 | head_mask=None, |
| 483 | encoder_hidden_states=None, |
| 484 | encoder_attention_mask=None, |
| 485 | past_key_values=None, |
| 486 | use_cache=None, |
| 487 | output_attentions=False, |
| 488 | output_hidden_states=False, |
| 489 | return_dict=True, |
| 490 | ): |
| 491 | all_hidden_states = () if output_hidden_states else None |
| 492 | all_self_attentions = () if output_attentions else None |
| 493 | all_cross_attentions = ( |
| 494 | ) if output_attentions and self.config.add_cross_attention else None |
| 495 | |
| 496 | next_decoder_cache = () if use_cache else None |
| 497 | for i, layer_module in enumerate(self.layer): |
| 498 | if output_hidden_states: |
| 499 | all_hidden_states = all_hidden_states + (hidden_states, ) |
| 500 | |
| 501 | layer_head_mask = head_mask[i] if head_mask is not None else None |
| 502 | past_key_value = past_key_values[ |
| 503 | i] if past_key_values is not None else None |
| 504 | |
| 505 | if self.gradient_checkpointing and self.training: |
| 506 | |
| 507 | if use_cache: |
| 508 | logger.warning( |
| 509 | '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' |
| 510 | ) |
| 511 | use_cache = False |
| 512 | |
| 513 | def create_custom_forward(module): |
| 514 | |
| 515 | def custom_forward(*inputs): |
| 516 | return module(*inputs, past_key_value, |
| 517 | output_attentions) |
| 518 | |
| 519 | return custom_forward |
| 520 | |
| 521 | layer_outputs = torch.utils.checkpoint.checkpoint( |
| 522 | create_custom_forward(layer_module), |
| 523 | hidden_states, |
| 524 | attention_mask, |
| 525 | layer_head_mask, |
| 526 | encoder_hidden_states, |
| 527 | encoder_attention_mask, |
| 528 | ) |
| 529 | else: |
| 530 | layer_outputs = layer_module( |
| 531 | hidden_states, |
| 532 | attention_mask, |
| 533 | layer_head_mask, |
| 534 | encoder_hidden_states, |
| 535 | encoder_attention_mask, |
nothing calls this directly
no test coverage detected