


"""simple docstring"""






from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFImageClassifierOutputWithNoAttention,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_resnet import ResNetConfig


_lowerCamelCase  =	logging.get_logger(__name__)

# General docstring
_lowerCamelCase  =	'ResNetConfig'

# Base docstring
_lowerCamelCase  =	'microsoft/resnet-50'
_lowerCamelCase  =	[1, 2048, 7, 7]

# Image classification docstring
_lowerCamelCase  =	'microsoft/resnet-50'
_lowerCamelCase  =	'tiger cat'

_lowerCamelCase  =	[
    'microsoft/resnet-50',
    # See all resnet models at https://huggingface.co/models?filter=resnet
]


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = 3							,		UpperCAmelCase__ = 1							,		UpperCAmelCase__ = "relu"							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					kernel_size // 2
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.ConvaD(
								    UpperCAmelCase_							,		kernel_size=UpperCAmelCase_							,		strides=UpperCAmelCase_							,		padding="valid"							,		use_bias=UpperCAmelCase_							,		name="convolution"  )
								# Use same default momentum and epsilon as PyTorch equivalent
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.BatchNormalization(epsilon=1e-5							,		momentum=0.9							,		name="normalization"  )
								SCREAMING_SNAKE_CASE__  				=					ACTaFN[activation] if activation is not None else tf.keras.layers.Activation("linear"  )



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					SCREAMING_SNAKE_CASE__  				=					(self.pad_value, self.pad_value)
								SCREAMING_SNAKE_CASE__  				=					tf.pad(UpperCAmelCase_							,		[(0, 0), height_pad, width_pad, (0, 0)]  )
								SCREAMING_SNAKE_CASE__  				=					self.conv(UpperCAmelCase_  )
								return hidden_state






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):
								SCREAMING_SNAKE_CASE__  				=					self.convolution(UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.normalization(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.activation(UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(
								    config.embedding_size							,		kernel_size=7							,		stride=2							,		activation=config.hidden_act							,		name="embedder"							,		)
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.MaxPoolaD(pool_size=3							,		strides=2							,		padding="valid"							,		name="pooler"  )
								SCREAMING_SNAKE_CASE__  				=					config.num_channels






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):
								SCREAMING_SNAKE_CASE__				,   SCREAMING_SNAKE_CASE__				,   SCREAMING_SNAKE_CASE__				,   SCREAMING_SNAKE_CASE__  				=					shape_list(UpperCAmelCase_  )
								if tf.executing_eagerly() and num_channels != self.num_channels:
												raise ValueError(
												    "Make sure that the channel dimension of the pixel values match with the one set in the configuration."  )
								SCREAMING_SNAKE_CASE__  				=					pixel_values
								SCREAMING_SNAKE_CASE__  				=					self.embedder(UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					tf.pad(UpperCAmelCase_							,		[[0, 0], [1, 1], [1, 1], [0, 0]]  )
								SCREAMING_SNAKE_CASE__  				=					self.pooler(UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = 2							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.ConvaD(
								    UpperCAmelCase_							,		kernel_size=1							,		strides=UpperCAmelCase_							,		use_bias=UpperCAmelCase_							,		name="convolution"  )
								# Use same default momentum and epsilon as PyTorch equivalent
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.BatchNormalization(epsilon=1e-5							,		momentum=0.9							,		name="normalization"  )






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):
								SCREAMING_SNAKE_CASE__  				=					x
								SCREAMING_SNAKE_CASE__  				=					self.convolution(UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.normalization(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__ = 1							,		UpperCAmelCase__ = "relu"							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					in_channels != out_channels or stride != 1
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(UpperCAmelCase_							,		stride=UpperCAmelCase_							,		name="layer.0"  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(UpperCAmelCase_							,		activation=UpperCAmelCase_							,		name="layer.1"  )
								SCREAMING_SNAKE_CASE__  				=					(
								    TFResNetShortCut(UpperCAmelCase_							,		stride=UpperCAmelCase_							,		name="shortcut"  )
								    if should_apply_shortcut
								    else tf.keras.layers.Activation("linear"							,		name="shortcut"  )
								)
								SCREAMING_SNAKE_CASE__  				=					ACTaFN[activation]






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):
								SCREAMING_SNAKE_CASE__  				=					hidden_state
								SCREAMING_SNAKE_CASE__  				=					self.conva(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.conva(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.shortcut(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								hidden_state += residual
								SCREAMING_SNAKE_CASE__  				=					self.activation(UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__ = 1							,		UpperCAmelCase__ = "relu"							,		UpperCAmelCase__ = 4							,		**UpperCAmelCase__							,		):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					in_channels != out_channels or stride != 1
								SCREAMING_SNAKE_CASE__  				=					out_channels // reduction
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(UpperCAmelCase_							,		kernel_size=1							,		name="layer.0"  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(UpperCAmelCase_							,		stride=UpperCAmelCase_							,		name="layer.1"  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetConvLayer(UpperCAmelCase_							,		kernel_size=1							,		activation=UpperCAmelCase_							,		name="layer.2"  )
								SCREAMING_SNAKE_CASE__  				=					(
								    TFResNetShortCut(UpperCAmelCase_							,		stride=UpperCAmelCase_							,		name="shortcut"  )
								    if should_apply_shortcut
								    else tf.keras.layers.Activation("linear"							,		name="shortcut"  )
								)
								SCREAMING_SNAKE_CASE__  				=					ACTaFN[activation]






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):
								SCREAMING_SNAKE_CASE__  				=					hidden_state
								SCREAMING_SNAKE_CASE__  				=					self.conva(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.conva(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.conva(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.shortcut(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								hidden_state += residual
								SCREAMING_SNAKE_CASE__  				=					self.activation(UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__							,		UpperCAmelCase__ = 2							,		UpperCAmelCase__ = 2							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )

								SCREAMING_SNAKE_CASE__  				=					TFResNetBottleNeckLayer if config.layer_type == "bottleneck" else TFResNetBasicLayer

								SCREAMING_SNAKE_CASE__  				=					[layer(UpperCAmelCase_							,		UpperCAmelCase_							,		stride=UpperCAmelCase_							,		activation=config.hidden_act							,		name="layers.0"  )]
								layers += [
								    layer(UpperCAmelCase_							,		UpperCAmelCase_							,		activation=config.hidden_act							,		name=f'''layers.{i + 1}'''  )
								    for i in range(depth - 1  )
								]
								SCREAMING_SNAKE_CASE__  				=					layers






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False  ):



								for layer in self.stage_layers:
												SCREAMING_SNAKE_CASE__  				=					layer(UpperCAmelCase_							,		training=UpperCAmelCase_  )
								return hidden_state


class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								# based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
								SCREAMING_SNAKE_CASE__  				=					[
								    TFResNetStage(
								        UpperCAmelCase_							,		config.embedding_size							,		config.hidden_sizes[0]							,		stride=2 if config.downsample_in_first_stage else 1							,		depth=config.depths[0]							,		name="stages.0"							,		)
								]
								for i, (in_channels, out_channels, depth) in enumerate(
								    zip(config.hidden_sizes							,		config.hidden_sizes[1:]							,		config.depths[1:]  )  ):
												self.stages.append(TFResNetStage(UpperCAmelCase_							,		UpperCAmelCase_							,		UpperCAmelCase_							,		depth=UpperCAmelCase_							,		name=f'''stages.{i + 1}'''  )  )






				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = False							,		UpperCAmelCase__ = True							,		UpperCAmelCase__ = False							,		):
								SCREAMING_SNAKE_CASE__  				=					() if output_hidden_states else None

								for stage_module in self.stages:
												if output_hidden_states:
																SCREAMING_SNAKE_CASE__  				=					hidden_states + (hidden_state,)

												SCREAMING_SNAKE_CASE__  				=					stage_module(UpperCAmelCase_							,		training=UpperCAmelCase_  )

								if output_hidden_states:
												SCREAMING_SNAKE_CASE__  				=					hidden_states + (hidden_state,)

								if not return_dict:
												return tuple(v for v in [hidden_state, hidden_states] if v is not None  )

								return TFBaseModelOutputWithNoAttention(last_hidden_state=UpperCAmelCase_							,		hidden_states=UpperCAmelCase_  )


class       lowerCamelCase_		(      snake_case__ ):





				"""simple docstring"""







				_lowerCAmelCase   :   int   				=       ResNetConfig
				_lowerCAmelCase   :   int   				=       """resnet"""
				_lowerCAmelCase   :   str   				=       """pixel_values"""



				@property
				def 	lowerCAmelCase__		(    self  ):
								return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 224, 224)							,		dtype=tf.floataa  )}


_lowerCamelCase  =	R'\n    This model is a TensorFlow\n    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a\n    regular TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.\n'


_lowerCamelCase  =	R'\n    Args:\n        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n'


@keras_serializable
class       lowerCamelCase_		(      tf.keras.layers.Layer ):





				"""simple docstring"""







				_lowerCAmelCase   :   List[str]   				=       ResNetConfig



				def __init__(    self							,		UpperCAmelCase__							,		**UpperCAmelCase__  ):
								super().__init__(**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					config
								SCREAMING_SNAKE_CASE__  				=					TFResNetEmbeddings(UpperCAmelCase_							,		name="embedder"  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetEncoder(UpperCAmelCase_							,		name="encoder"  )
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.GlobalAveragePoolingaD(keepdims=UpperCAmelCase_  )






				@unpack_inputs
				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = False							,		):
								SCREAMING_SNAKE_CASE__  				=					(
								    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
								)
								SCREAMING_SNAKE_CASE__  				=					return_dict if return_dict is not None else self.config.use_return_dict

								# TF 2.0 image layers can't use NCHW format when running on CPU.
								# We transpose to NHWC format and then transpose back after the full forward pass.
								# (batch_size, num_channels, height, width) -> (batch_size, height, width, num_channels)
								SCREAMING_SNAKE_CASE__  				=					tf.transpose(UpperCAmelCase_							,		perm=[0, 2, 3, 1]  )
								SCREAMING_SNAKE_CASE__  				=					self.embedder(UpperCAmelCase_							,		training=UpperCAmelCase_  )

								SCREAMING_SNAKE_CASE__  				=					self.encoder(
								    UpperCAmelCase_							,		output_hidden_states=UpperCAmelCase_							,		return_dict=UpperCAmelCase_							,		training=UpperCAmelCase_  )

								SCREAMING_SNAKE_CASE__  				=					encoder_outputs[0]

								SCREAMING_SNAKE_CASE__  				=					self.pooler(UpperCAmelCase_  )

								# Transpose all the outputs to the NCHW format
								# (batch_size, height, width, num_channels) -> (batch_size, num_channels, height, width)
								SCREAMING_SNAKE_CASE__  				=					tf.transpose(UpperCAmelCase_							,		(0, 3, 1, 2)  )
								SCREAMING_SNAKE_CASE__  				=					tf.transpose(UpperCAmelCase_							,		(0, 3, 1, 2)  )
								SCREAMING_SNAKE_CASE__  				=					()
								for hidden_state in encoder_outputs[1:]:
												SCREAMING_SNAKE_CASE__  				=					hidden_states + tuple(tf.transpose(UpperCAmelCase_							,		(0, 3, 1, 2)  ) for h in hidden_state  )

								if not return_dict:
												return (last_hidden_state, pooled_output) + hidden_states

								SCREAMING_SNAKE_CASE__  				=					hidden_states if output_hidden_states else None

								return TFBaseModelOutputWithPoolingAndNoAttention(
								    last_hidden_state=UpperCAmelCase_							,		pooler_output=UpperCAmelCase_							,		hidden_states=UpperCAmelCase_							,		)


@add_start_docstrings(
    "The bare ResNet model outputting raw features without any specific head on top."							,	snake_case__							,	)
class       lowerCamelCase_		(      snake_case__ ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		**UpperCAmelCase__  ):
								super().__init__(UpperCAmelCase_							,		**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					TFResNetMainLayer(config=UpperCAmelCase_							,		name="resnet"  )






				@add_start_docstrings_to_model_forward(UpperCAmelCase_  )
				@add_code_sample_docstrings(
				    checkpoint=_CHECKPOINT_FOR_DOC							,		output_type=UpperCAmelCase_							,		config_class=_CONFIG_FOR_DOC							,		modality="vision"							,		expected_output=_EXPECTED_OUTPUT_SHAPE							,		)
				@unpack_inputs
				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = False							,		):
								SCREAMING_SNAKE_CASE__  				=					(
								    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
								)
								SCREAMING_SNAKE_CASE__  				=					return_dict if return_dict is not None else self.config.use_return_dict

								SCREAMING_SNAKE_CASE__  				=					self.resnet(
								    pixel_values=UpperCAmelCase_							,		output_hidden_states=UpperCAmelCase_							,		return_dict=UpperCAmelCase_							,		training=UpperCAmelCase_							,		)
								return resnet_outputs


@add_start_docstrings(
    "\n    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "							,	snake_case__							,	)
class       lowerCamelCase_		(      snake_case__							,	snake_case__ ):





				"""simple docstring"""







				def __init__(    self							,		UpperCAmelCase__							,		**UpperCAmelCase__  ):
								super().__init__(UpperCAmelCase_							,		**UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					config.num_labels
								SCREAMING_SNAKE_CASE__  				=					TFResNetMainLayer(UpperCAmelCase_							,		name="resnet"  )
								# classification head
								SCREAMING_SNAKE_CASE__  				=					(
								    tf.keras.layers.Dense(config.num_labels							,		name="classifier.1"  )
								    if config.num_labels > 0
								    else tf.keras.layers.Activation("linear"							,		name="classifier.1"  )
								)



				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__  ):
								SCREAMING_SNAKE_CASE__  				=					tf.keras.layers.Flatten()(UpperCAmelCase_  )
								SCREAMING_SNAKE_CASE__  				=					self.classifier_layer(UpperCAmelCase_  )
								return logits






				@add_start_docstrings_to_model_forward(UpperCAmelCase_  )
				@add_code_sample_docstrings(
				    checkpoint=_IMAGE_CLASS_CHECKPOINT							,		output_type=UpperCAmelCase_							,		config_class=_CONFIG_FOR_DOC							,		expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT							,		)
				@unpack_inputs
				def 	lowerCAmelCase__		(    self							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = None							,		UpperCAmelCase__ = False							,		):
								SCREAMING_SNAKE_CASE__  				=					return_dict if return_dict is not None else self.config.use_return_dict

								SCREAMING_SNAKE_CASE__  				=					self.resnet(
								    UpperCAmelCase_							,		output_hidden_states=UpperCAmelCase_							,		return_dict=UpperCAmelCase_							,		training=UpperCAmelCase_  )

								SCREAMING_SNAKE_CASE__  				=					outputs.pooler_output if return_dict else outputs[1]

								SCREAMING_SNAKE_CASE__  				=					self.classifier(UpperCAmelCase_  )

								SCREAMING_SNAKE_CASE__  				=					None if labels is None else self.hf_compute_loss(UpperCAmelCase_							,		UpperCAmelCase_  )

								if not return_dict:
												SCREAMING_SNAKE_CASE__  				=					(logits,) + outputs[2:]
												return (loss,) + output if loss is not None else output

								return TFImageClassifierOutputWithNoAttention(loss=UpperCAmelCase_							,		logits=UpperCAmelCase_							,		hidden_states=outputs.hidden_states  )




"""simple docstring"""
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class   __A  (snake_case__    ,		snake_case__    ,		snake_case__):





							'''simple docstring'''

							@register_to_config
							def __init__(					self				:   List[str]		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   float		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   int		,							UpperCAmelCase_				:   str		,							UpperCAmelCase_				:   bool = False		,							)       ->Dict:



														"""simple docstring"""



														super().__init__()

														snake_case_     							=		nn.Linear(UpperCAmelCase_		,							UpperCAmelCase_		,							bias=UpperCAmelCase_							)

														snake_case_     							=		nn.Embedding(UpperCAmelCase_		,							UpperCAmelCase_							)
														snake_case_     							=		False

														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)

														snake_case_     							=		TaConfig(
														    d_model=UpperCAmelCase_		,							num_heads=UpperCAmelCase_		,							d_kv=UpperCAmelCase_		,							d_ff=UpperCAmelCase_		,							feed_forward_proj=UpperCAmelCase_		,							dropout_rate=UpperCAmelCase_		,							is_decoder=UpperCAmelCase_		,							is_encoder_decoder=UpperCAmelCase_		,							)
														snake_case_     							=		nn.ModuleList()
														for lyr_num in range(UpperCAmelCase_							):
																					snake_case_     							=		TaBlock(UpperCAmelCase_							)
																					self.encoders.append(UpperCAmelCase_							)

														snake_case_     							=		TaLayerNorm(UpperCAmelCase_							)
														snake_case_     							=		nn.Dropout(p=UpperCAmelCase_							)



							def 		lowerCAmelCase							(					self				:   Optional[Any]		,							UpperCAmelCase_				:   Any		,							UpperCAmelCase_				:   List[str]							)       ->List[Any]:



														"""simple docstring"""



														snake_case_     							=		self.input_proj(UpperCAmelCase_							)

														# terminal relative positional encodings
														snake_case_     							=		encoder_inputs.shape[1]
														snake_case_     							=		torch.arange(UpperCAmelCase_		,							device=encoder_inputs.device							)

														snake_case_     							=		encoder_inputs_mask.sum(-1							)
														snake_case_     							=		torch.roll(input_positions.unsqueeze(0							)		,							tuple(seq_lens.tolist()							)		,							dims=0							)
														x += self.position_encoding(UpperCAmelCase_							)

														snake_case_     							=		self.dropout_pre(UpperCAmelCase_							)

														# inverted the attention mask
														snake_case_     							=		encoder_inputs.size()
														snake_case_     							=		self.get_extended_attention_mask(UpperCAmelCase_		,							UpperCAmelCase_							)

														for lyr in self.encoders:
																					snake_case_     							=		lyr(UpperCAmelCase_		,							UpperCAmelCase_							)[0]
														snake_case_     							=		self.layer_norm(UpperCAmelCase_							)

														return self.dropout_post(UpperCAmelCase_							), encoder_inputs_mask



