(
self,
hidden_states,
attention_mask=None,
head_mask=None,
encoder_hidden_states=None,
encoder_attention_mask=None,
past_key_values=None,
use_cache=None,
output_attentions=False,
output_hidden_states=False,
return_dict=True,
)
| 479 | self.gradient_checkpointing = False |
| 480 | |
| 481 | def forward( |
| 482 | self, |
| 483 | hidden_states, |
| 484 | attention_mask=None, |
| 485 | head_mask=None, |
| 486 | encoder_hidden_states=None, |
| 487 | encoder_attention_mask=None, |
| 488 | past_key_values=None, |
| 489 | use_cache=None, |
| 490 | output_attentions=False, |
| 491 | output_hidden_states=False, |
| 492 | return_dict=True, |
| 493 | ): |
| 494 | all_hidden_states = () if output_hidden_states else None |
| 495 | all_self_attentions = () if output_attentions else None |
| 496 | all_cross_attentions = ( |
| 497 | ) if output_attentions and self.config.add_cross_attention else None |
| 498 | |
| 499 | next_decoder_cache = () if use_cache else None |
| 500 | for i, layer_module in enumerate(self.layer): |
| 501 | if output_hidden_states: |
| 502 | all_hidden_states = all_hidden_states + (hidden_states, ) |
| 503 | |
| 504 | layer_head_mask = head_mask[i] if head_mask is not None else None |
| 505 | past_key_value = past_key_values[ |
| 506 | i] if past_key_values is not None else None |
| 507 | |
| 508 | if self.gradient_checkpointing and self.training: |
| 509 | |
| 510 | if use_cache: |
| 511 | logger.warning( |
| 512 | '`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...' |
| 513 | ) |
| 514 | use_cache = False |
| 515 | |
| 516 | def create_custom_forward(module): |
| 517 | |
| 518 | def custom_forward(*inputs): |
| 519 | return module(*inputs, past_key_value, |
| 520 | output_attentions) |
| 521 | |
| 522 | return custom_forward |
| 523 | |
| 524 | layer_outputs = torch.utils.checkpoint.checkpoint( |
| 525 | create_custom_forward(layer_module), |
| 526 | hidden_states, |
| 527 | attention_mask, |
| 528 | layer_head_mask, |
| 529 | encoder_hidden_states, |
| 530 | encoder_attention_mask, |
| 531 | ) |
| 532 | else: |
| 533 | layer_outputs = layer_module( |
| 534 | hidden_states, |
| 535 | attention_mask, |
| 536 | layer_head_mask, |
| 537 | encoder_hidden_states, |
| 538 | encoder_attention_mask, |
nothing calls this directly
no test coverage detected