


"""simple docstring"""
from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFImageClassifierOutputWithNoAttention,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_resnet import ResNetConfig


__SCREAMING_SNAKE_CASE				:     List[str]		=     logging.get_logger(__name__)

# General docstring
__SCREAMING_SNAKE_CASE				:     List[Any]		=     'ResNetConfig'

# Base docstring
__SCREAMING_SNAKE_CASE				:     Dict		=     'microsoft/resnet-50'
__SCREAMING_SNAKE_CASE				:     List[str]		=     [1, 2_048, 7, 7]

# Image classification docstring
__SCREAMING_SNAKE_CASE				:     Optional[int]		=     'microsoft/resnet-50'
__SCREAMING_SNAKE_CASE				:     Tuple		=     'tiger cat'

__SCREAMING_SNAKE_CASE				:     List[Any]		=     [
    'microsoft/resnet-50',
    # See all resnet models at https://huggingface.co/models?filter=resnet
]





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Any		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int = 3		,							UpperCAmelCase_				:   int = 1		,							UpperCAmelCase_				:   str = "relu"		,							**UpperCAmelCase_				:   str							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		kernel_size // 2
														snake_case_     							=		tf.keras.layers.ConvaD(
														    UpperCAmelCase_		,							kernel_size=UpperCAmelCase_		,							strides=UpperCAmelCase_		,							padding="""valid"""		,							use_bias=UpperCAmelCase_		,							name="""convolution"""							)
														# Use same default momentum and epsilon as PyTorch equivalent
														snake_case_     							=		tf.keras.layers.BatchNormalization(epsilon=1E-5		,							momentum=0.9		,							name="""normalization"""							)
														snake_case_     							=		ACTaFN[activation] if activation is not None else tf.keras.layers.Activation("""linear"""							)



							def 		lowerCAmelCase							(					self				:   Tuple		,							UpperCAmelCase_				:   tf.Tensor							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		snake_case_     							=		(self.pad_value, self.pad_value)
														snake_case_     							=		tf.pad(UpperCAmelCase_		,							[(0, 0), height_pad, width_pad, (0, 0)]							)
														snake_case_     							=		self.conv(UpperCAmelCase_							)
														return hidden_state



							def 		lowerCAmelCase							(					self				:   int		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		self.convolution(UpperCAmelCase_							)
														snake_case_     							=		self.normalization(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.activation(UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Optional[Any]		,							UpperCAmelCase_				:   ResNetConfig		,							**UpperCAmelCase_				:   List[str]							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		TFResNetConvLayer(
														    config.embedding_size		,							kernel_size=7		,							stride=2		,							activation=config.hidden_act		,							name="""embedder"""		,							)
														snake_case_     							=		tf.keras.layers.MaxPoolaD(pool_size=3		,							strides=2		,							padding="""valid"""		,							name="""pooler"""							)
														snake_case_     							=		config.num_channels



							def 		lowerCAmelCase							(					self				:   int		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_							,    snake_case_							,    snake_case_							,    snake_case_     							=		shape_list(UpperCAmelCase_							)
														if tf.executing_eagerly() and num_channels != self.num_channels:
																					raise ValueError(
																					    """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""							)
														snake_case_     							=		pixel_values
														snake_case_     							=		self.embedder(UpperCAmelCase_							)
														snake_case_     							=		tf.pad(UpperCAmelCase_		,							[[0, 0], [1, 1], [1, 1], [0, 0]]							)
														snake_case_     							=		self.pooler(UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   List[Any]		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int = 2		,							**UpperCAmelCase_				:   List[str]							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		tf.keras.layers.ConvaD(
														    UpperCAmelCase_		,							kernel_size=1		,							strides=UpperCAmelCase_		,							use_bias=UpperCAmelCase_		,							name="""convolution"""							)
														# Use same default momentum and epsilon as PyTorch equivalent
														snake_case_     							=		tf.keras.layers.BatchNormalization(epsilon=1E-5		,							momentum=0.9		,							name="""normalization"""							)



							def 		lowerCAmelCase							(					self				:   Optional[int]		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		x
														snake_case_     							=		self.convolution(UpperCAmelCase_							)
														snake_case_     							=		self.normalization(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Union[str, Any]		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int = 1		,							UpperCAmelCase_				:   str = "relu"		,							**UpperCAmelCase_				:   List[Any]							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		in_channels != out_channels or stride != 1
														snake_case_     							=		TFResNetConvLayer(UpperCAmelCase_		,							stride=UpperCAmelCase_		,							name="""layer.0"""							)
														snake_case_     							=		TFResNetConvLayer(UpperCAmelCase_		,							activation=UpperCAmelCase_		,							name="""layer.1"""							)
														snake_case_     							=		(
														    TFResNetShortCut(UpperCAmelCase_		,							stride=UpperCAmelCase_		,							name="""shortcut"""							)
														    if should_apply_shortcut
														    else tf.keras.layers.Activation("""linear"""		,							name="""shortcut"""							)
														)
														snake_case_     							=		ACTaFN[activation]



							def 		lowerCAmelCase							(					self				:   str		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		hidden_state
														snake_case_     							=		self.conva(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.conva(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.shortcut(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														hidden_state += residual
														snake_case_     							=		self.activation(UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Any		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int = 1		,							UpperCAmelCase_				:   str = "relu"		,							UpperCAmelCase_				:   int = 4		,							**UpperCAmelCase_				:   List[str]		,							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		in_channels != out_channels or stride != 1
														snake_case_     							=		out_channels // reduction
														snake_case_     							=		TFResNetConvLayer(UpperCAmelCase_		,							kernel_size=1		,							name="""layer.0"""							)
														snake_case_     							=		TFResNetConvLayer(UpperCAmelCase_		,							stride=UpperCAmelCase_		,							name="""layer.1"""							)
														snake_case_     							=		TFResNetConvLayer(UpperCAmelCase_		,							kernel_size=1		,							activation=UpperCAmelCase_		,							name="""layer.2"""							)
														snake_case_     							=		(
														    TFResNetShortCut(UpperCAmelCase_		,							stride=UpperCAmelCase_		,							name="""shortcut"""							)
														    if should_apply_shortcut
														    else tf.keras.layers.Activation("""linear"""		,							name="""shortcut"""							)
														)
														snake_case_     							=		ACTaFN[activation]



							def 		lowerCAmelCase							(					self				:   Optional[Any]		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		hidden_state
														snake_case_     							=		self.conva(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.conva(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.conva(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														snake_case_     							=		self.shortcut(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														hidden_state += residual
														snake_case_     							=		self.activation(UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Optional[int]		,							UpperCAmelCase_				:   ResNetConfig		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int = 2		,							UpperCAmelCase_				:   int = 2		,							**UpperCAmelCase_				:   Any							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)

														snake_case_     							=		TFResNetBottleNeckLayer if config.layer_type == """bottleneck""" else TFResNetBasicLayer

														snake_case_     							=		[layer(UpperCAmelCase_		,							UpperCAmelCase_		,							stride=UpperCAmelCase_		,							activation=config.hidden_act		,							name="""layers.0"""							)]
														layers += [
														    layer(UpperCAmelCase_		,							UpperCAmelCase_		,							activation=config.hidden_act		,							name=F"""layers.{i + 1}"""							)
														    for i in range(depth - 1							)
														]
														snake_case_     							=		layers



							def 		lowerCAmelCase							(					self				:   Union[str, Any]		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False							)       ->tf.Tensor:



														"""simple docstring"""



														for layer in self.stage_layers:
																					snake_case_     							=		layer(UpperCAmelCase_		,							training=UpperCAmelCase_							)
														return hidden_state





class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							def __init__(					self				:   Tuple		,							UpperCAmelCase_				:   ResNetConfig		,							**UpperCAmelCase_				:   int							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														# based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
														snake_case_     							=		[
														    TFResNetStage(
														        UpperCAmelCase_		,							config.embedding_size		,							config.hidden_sizes[0]		,							stride=2 if config.downsample_in_first_stage else 1		,							depth=config.depths[0]		,							name="""stages.0"""		,							)
														]
														for i, (in_channels, out_channels, depth) in enumerate(
														    zip(config.hidden_sizes		,							config.hidden_sizes[1:]		,							config.depths[1:]							)							):
																					self.stages.append(TFResNetStage(UpperCAmelCase_		,							UpperCAmelCase_		,							UpperCAmelCase_		,							depth=UpperCAmelCase_		,							name=F"""stages.{i + 1}"""							)							)



							def 		lowerCAmelCase							(					self				:   Optional[int]		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   bool = False		,							UpperCAmelCase_				:   bool = True		,							UpperCAmelCase_				:   bool = False		,							)       ->TFBaseModelOutputWithNoAttention:



														"""simple docstring"""



														snake_case_     							=		() if output_hidden_states else None

														for stage_module in self.stages:
																					if output_hidden_states:
																												snake_case_     							=		hidden_states + (hidden_state,)

																					snake_case_     							=		stage_module(UpperCAmelCase_		,							training=UpperCAmelCase_							)

														if output_hidden_states:
																					snake_case_     							=		hidden_states + (hidden_state,)

														if not return_dict:
																					return tuple(v for v in [hidden_state, hidden_states] if v is not None							)

														return TFBaseModelOutputWithNoAttention(last_hidden_state=UpperCAmelCase_		,							hidden_states=UpperCAmelCase_							)





class   __A  (snake_case__):





							'''simple docstring'''

							__lowercase:		int				   =     ResNetConfig
							__lowercase:		int				   =     """resnet"""
							__lowercase:		str				   =     """pixel_values"""



							@property
							def 		lowerCAmelCase							(					self				:   List[Any]							)       ->Tuple:



														"""simple docstring"""



														return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 224, 224)		,							dtype=tf.floataa							)}


__SCREAMING_SNAKE_CASE				:     Tuple		=     R'\n    This model is a TensorFlow\n    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a\n    regular TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.\n'


__SCREAMING_SNAKE_CASE				:     Dict		=     R'\n    Args:\n        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'





@keras_serializable
class   __A  (tf.keras.layers.Layer):





							'''simple docstring'''

							__lowercase:		List[str]				   =     ResNetConfig



							def __init__(					self				:   Any		,							UpperCAmelCase_				:   ResNetConfig		,							**UpperCAmelCase_				:   Tuple							)       ->None:



														"""simple docstring"""



														super().__init__(**UpperCAmelCase_							)
														snake_case_     							=		config
														snake_case_     							=		TFResNetEmbeddings(UpperCAmelCase_		,							name="""embedder"""							)
														snake_case_     							=		TFResNetEncoder(UpperCAmelCase_		,							name="""encoder"""							)
														snake_case_     							=		tf.keras.layers.GlobalAveragePoolingaD(keepdims=UpperCAmelCase_							)



							@unpack_inputs
							def 		lowerCAmelCase							(					self				:   Any		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   Optional[bool] = None		,							UpperCAmelCase_				:   Optional[bool] = None		,							UpperCAmelCase_				:   bool = False		,							)       ->Union[Tuple[tf.Tensor], TFBaseModelOutputWithPoolingAndNoAttention]:



														"""simple docstring"""



														snake_case_     							=		(
														    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
														)
														snake_case_     							=		return_dict if return_dict is not None else self.config.use_return_dict

														# TF 2.0 image layers can't use NCHW format when running on CPU.
														# We transpose to NHWC format and then transpose back after the full forward pass.
														# (batch_size, num_channels, height, width) -> (batch_size, height, width, num_channels)
														snake_case_     							=		tf.transpose(UpperCAmelCase_		,							perm=[0, 2, 3, 1]							)
														snake_case_     							=		self.embedder(UpperCAmelCase_		,							training=UpperCAmelCase_							)

														snake_case_     							=		self.encoder(
														    UpperCAmelCase_		,							output_hidden_states=UpperCAmelCase_		,							return_dict=UpperCAmelCase_		,							training=UpperCAmelCase_							)

														snake_case_     							=		encoder_outputs[0]

														snake_case_     							=		self.pooler(UpperCAmelCase_							)

														# Transpose all the outputs to the NCHW format
														# (batch_size, height, width, num_channels) -> (batch_size, num_channels, height, width)
														snake_case_     							=		tf.transpose(UpperCAmelCase_		,							(0, 3, 1, 2)							)
														snake_case_     							=		tf.transpose(UpperCAmelCase_		,							(0, 3, 1, 2)							)
														snake_case_     							=		()
														for hidden_state in encoder_outputs[1:]:
																					snake_case_     							=		hidden_states + tuple(tf.transpose(UpperCAmelCase_		,							(0, 3, 1, 2)							) for h in hidden_state							)

														if not return_dict:
																					return (last_hidden_state, pooled_output) + hidden_states

														snake_case_     							=		hidden_states if output_hidden_states else None

														return TFBaseModelOutputWithPoolingAndNoAttention(
														    last_hidden_state=UpperCAmelCase_		,							pooler_output=UpperCAmelCase_		,							hidden_states=UpperCAmelCase_		,							)





@add_start_docstrings(
    """The bare ResNet model outputting raw features without any specific head on top."""    ,		snake_case__    ,		)
class   __A  (snake_case__):





							'''simple docstring'''

							def __init__(					self				:   int		,							UpperCAmelCase_				:   ResNetConfig		,							**UpperCAmelCase_				:   List[Any]							)       ->None:



														"""simple docstring"""



														super().__init__(UpperCAmelCase_		,							**UpperCAmelCase_							)
														snake_case_     							=		TFResNetMainLayer(config=UpperCAmelCase_		,							name="""resnet"""							)



							@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
							@add_code_sample_docstrings(
							    checkpoint=_CHECKPOINT_FOR_DOC		,							output_type=UpperCAmelCase_		,							config_class=_CONFIG_FOR_DOC		,							modality="""vision"""		,							expected_output=_EXPECTED_OUTPUT_SHAPE		,							)
							@unpack_inputs
							def 		lowerCAmelCase							(					self				:   Dict		,							UpperCAmelCase_				:   tf.Tensor		,							UpperCAmelCase_				:   Optional[bool] = None		,							UpperCAmelCase_				:   Optional[bool] = None		,							UpperCAmelCase_				:   bool = False		,							)       ->Union[Tuple[tf.Tensor], TFBaseModelOutputWithPoolingAndNoAttention]:



														"""simple docstring"""



														snake_case_     							=		(
														    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
														)
														snake_case_     							=		return_dict if return_dict is not None else self.config.use_return_dict

														snake_case_     							=		self.resnet(
														    pixel_values=UpperCAmelCase_		,							output_hidden_states=UpperCAmelCase_		,							return_dict=UpperCAmelCase_		,							training=UpperCAmelCase_		,							)
														return resnet_outputs





@add_start_docstrings(
    """
    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    """    ,		snake_case__    ,		)
class   __A  (snake_case__    ,		snake_case__):





							'''simple docstring'''

							def __init__(					self				:   Any		,							UpperCAmelCase_				:   ResNetConfig		,							**UpperCAmelCase_				:   List[str]							)       ->None:



														"""simple docstring"""



														super().__init__(UpperCAmelCase_		,							**UpperCAmelCase_							)
														snake_case_     							=		config.num_labels
														snake_case_     							=		TFResNetMainLayer(UpperCAmelCase_		,							name="""resnet"""							)
														# classification head
														snake_case_     							=		(
														    tf.keras.layers.Dense(config.num_labels		,							name="""classifier.1"""							)
														    if config.num_labels > 0
														    else tf.keras.layers.Activation("""linear"""		,							name="""classifier.1"""							)
														)



							def 		lowerCAmelCase							(					self				:   int		,							UpperCAmelCase_				:   tf.Tensor							)       ->tf.Tensor:



														"""simple docstring"""



														snake_case_     							=		tf.keras.layers.Flatten()(UpperCAmelCase_							)
														snake_case_     							=		self.classifier_layer(UpperCAmelCase_							)
														return logits



							@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
							@add_code_sample_docstrings(
							    checkpoint=_IMAGE_CLASS_CHECKPOINT		,							output_type=UpperCAmelCase_		,							config_class=_CONFIG_FOR_DOC		,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT		,							)
							@unpack_inputs
							def 		lowerCAmelCase							(					self				:   str		,							UpperCAmelCase_				:   tf.Tensor = None		,							UpperCAmelCase_				:   tf.Tensor = None		,							UpperCAmelCase_				:   bool = None		,							UpperCAmelCase_				:   bool = None		,							UpperCAmelCase_				:   bool = False		,							)       ->Union[Tuple[tf.Tensor], TFImageClassifierOutputWithNoAttention]:



														"""simple docstring"""



														snake_case_     							=		return_dict if return_dict is not None else self.config.use_return_dict

														snake_case_     							=		self.resnet(
														    UpperCAmelCase_		,							output_hidden_states=UpperCAmelCase_		,							return_dict=UpperCAmelCase_		,							training=UpperCAmelCase_							)

														snake_case_     							=		outputs.pooler_output if return_dict else outputs[1]

														snake_case_     							=		self.classifier(UpperCAmelCase_							)

														snake_case_     							=		None if labels is None else self.hf_compute_loss(UpperCAmelCase_		,							UpperCAmelCase_							)

														if not return_dict:
																					snake_case_     							=		(logits,) + outputs[2:]
																					return (loss,) + output if loss is not None else output

														return TFImageClassifierOutputWithNoAttention(loss=UpperCAmelCase_		,							logits=UpperCAmelCase_		,							hidden_states=outputs.hidden_states							)







"""simple docstring"""
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class   __A  (snake_case__    ,		snake_case__    ,		snake_case__):





							'''simple docstring'''

							@register_to_config
							def __init__(					self				:   List[str]		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   float		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   str		,							UpperCAmelCase_				:   bool = False		,							)       ->Dict:



														"""simple docstring"""



														super().__init__()

														snake_case_     							=		nn.Linear(UpperCAmelCase_		,							UpperCAmelCase_		,							bias=UpperCAmelCase_							)

														snake_case_     							=		nn.Embedding(UpperCAmelCase_		,							UpperCAmelCase_							)
														snake_case_     							=		False

														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)

														snake_case_     							=		TaConfig(
														    d_model=UpperCAmelCase_		,							num_heads=UpperCAmelCase_		,							d_kv=UpperCAmelCase_		,							d_ff=UpperCAmelCase_		,							feed_forward_proj=UpperCAmelCase_		,							dropout_rate=UpperCAmelCase_		,							is_decoder=UpperCAmelCase_		,							is_encoder_decoder=UpperCAmelCase_		,							)
														snake_case_     							=		nn.ModuleList()
														for lyr_num in range(UpperCAmelCase_							):
																					snake_case_     							=		TaBlock(UpperCAmelCase_							)
																					self.encoders.append(UpperCAmelCase_							)

														snake_case_     							=		TaLayerNorm(UpperCAmelCase_							)
														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)



							def 		lowerCAmelCase							(					self				:   Optional[Any]		,							UpperCAmelCase_				:   Any		,							UpperCAmelCase_				:   List[str]							)       ->List[Any]:



														"""simple docstring"""



														snake_case_     							=		self.input_proj(UpperCAmelCase_							)

														# terminal relative positional encodings
														snake_case_     							=		encoder_inputs.shape[1]
														snake_case_     							=		torch.arange(UpperCAmelCase_		,							device=encoder_inputs.device							)

														snake_case_     							=		encoder_inputs_mask.sum(-1							)
														snake_case_     							=		torch.roll(input_positions.unsqueeze(0							)		,							tuple(seq_lens.tolist()							)		,							dims=0							)
														x += self.position_encoding(UpperCAmelCase_							)

														snake_case_     							=		self.dropout_pre(UpperCAmelCase_							)

														# inverted the attention mask
														snake_case_     							=		encoder_inputs.size()
														snake_case_     							=		self.get_extended_attention_mask(UpperCAmelCase_		,							UpperCAmelCase_							)

														for lyr in self.encoders:
																					snake_case_     							=		lyr(UpperCAmelCase_		,							UpperCAmelCase_							)[0]
														snake_case_     							=		self.layer_norm(UpperCAmelCase_							)

														return self.dropout_post(UpperCAmelCase_							), encoder_inputs_mask



