





"""simple docstring"""




import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class 					_a		( _lowerCAmelCase      ,							_lowerCAmelCase      ,							_lowerCAmelCase							):






				@register_to_config
				def __init__(					self						:       List[Any],					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       float,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       int,					lowerCAmelCase__						:       str,					lowerCAmelCase__						:       bool = False,					)      -> str:




										'''simple docstring'''



										super().__init__()

										_UpperCamelCase     :   List[str]      =       nn.Linear(lowerCAmelCase__,					lowerCAmelCase__,					bias=lowerCAmelCase__				)

										_UpperCamelCase     :   List[Any]      =       nn.Embedding(lowerCAmelCase__,					lowerCAmelCase__				)
										_UpperCamelCase     :   Optional[Any]      =       False

										_UpperCamelCase     :   str      =       nn.Dropout(p=lowerCAmelCase__				)

										_UpperCamelCase     :   str      =       TaConfig(
										    d_model=lowerCAmelCase__,					num_heads=lowerCAmelCase__,					d_kv=lowerCAmelCase__,					d_ff=lowerCAmelCase__,					feed_forward_proj=lowerCAmelCase__,					dropout_rate=lowerCAmelCase__,					is_decoder=lowerCAmelCase__,					is_encoder_decoder=lowerCAmelCase__,					)
										_UpperCamelCase     :   List[str]      =       nn.ModuleList()
										for lyr_num in range(lowerCAmelCase__				):
																_UpperCamelCase     :   Any      =       TaBlock(lowerCAmelCase__				)
																self.encoders.append(lowerCAmelCase__				)

										_UpperCamelCase     :   int      =       TaLayerNorm(lowerCAmelCase__				)
										_UpperCamelCase     :   List[Any]      =       nn.Dropout(p=lowerCAmelCase__				)






				def 							snake_case      (					self						:       Union[str, Any],					lowerCAmelCase__						:       Dict,					lowerCAmelCase__						:       int				)      -> List[str]:




										'''simple docstring'''



										_UpperCamelCase     :   List[Any]      =       self.input_proj(lowerCAmelCase__				)

										# terminal relative positional encodings
										_UpperCamelCase     :   Optional[int]      =       encoder_inputs.shape[1]
										_UpperCamelCase     :   Tuple      =       torch.arange(lowerCAmelCase__,					device=encoder_inputs.device				)

										_UpperCamelCase     :   Union[str, Any]      =       encoder_inputs_mask.sum(-1				)
										_UpperCamelCase     :   List[str]      =       torch.roll(input_positions.unsqueeze(0				),					tuple(seq_lens.tolist()				),					dims=0				)
										x += self.position_encoding(lowerCAmelCase__				)

										_UpperCamelCase     :   List[str]      =       self.dropout_pre(lowerCAmelCase__				)

										# inverted the attention mask
										_UpperCamelCase     :   Any      =       encoder_inputs.size()
										_UpperCamelCase     :   Any      =       self.get_extended_attention_mask(lowerCAmelCase__,					lowerCAmelCase__				)

										for lyr in self.encoders:
																_UpperCamelCase     :   Union[str, Any]      =       lyr(lowerCAmelCase__,					lowerCAmelCase__				)[0]
										_UpperCamelCase     :   Union[str, Any]      =       self.layer_norm(lowerCAmelCase__				)

										return self.dropout_post(lowerCAmelCase__				), encoder_inputs_mask





"""simple docstring"""


from ..utils import DummyObject, requires_backends


__A					=			None



def        _lowerCamelCase(*__UpperCamelCase       ,					**__UpperCamelCase			)							->					Tuple:
	requires_backends(__UpperCamelCase       ,					["""sentencepiece""", """tokenizers"""]			)



