






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class __lowerCamelCase (      snake_case__     ,       snake_case__     ,       snake_case__):





	"""simple docstring"""






	@register_to_config
	def __init__(							self       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase       ,    UpperCAmelCase = False       ,    ):
							"""simple docstring"""







							super().__init__()

							_UpperCAmelCase									=			nn.Embedding(UpperCAmelCase       ,    UpperCAmelCase    )

							_UpperCAmelCase									=			nn.Embedding(UpperCAmelCase       ,    UpperCAmelCase    )
							_UpperCAmelCase									=			False

							_UpperCAmelCase									=			nn.Dropout(p=UpperCAmelCase    )

							_UpperCAmelCase									=			TaConfig(
							    vocab_size=UpperCAmelCase       ,    d_model=UpperCAmelCase       ,    num_heads=UpperCAmelCase       ,    d_kv=UpperCAmelCase       ,    d_ff=UpperCAmelCase       ,    dropout_rate=UpperCAmelCase       ,    feed_forward_proj=UpperCAmelCase       ,    is_decoder=UpperCAmelCase       ,    is_encoder_decoder=UpperCAmelCase       ,    )

							_UpperCAmelCase									=			nn.ModuleList()
							for lyr_num in range(UpperCAmelCase    ):
													_UpperCAmelCase									=			TaBlock(UpperCAmelCase    )
													self.encoders.append(UpperCAmelCase    )

							_UpperCAmelCase									=			TaLayerNorm(UpperCAmelCase    )
							_UpperCAmelCase									=			nn.Dropout(p=UpperCAmelCase    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.token_embedder(UpperCAmelCase    )

							_UpperCAmelCase									=			encoder_input_tokens.shape[1]
							_UpperCAmelCase									=			torch.arange(UpperCAmelCase       ,    device=encoder_input_tokens.device    )
							x += self.position_encoding(UpperCAmelCase    )

							_UpperCAmelCase									=			self.dropout_pre(UpperCAmelCase    )

							# inverted the attention mask
							_UpperCAmelCase									=			encoder_input_tokens.size()
							_UpperCAmelCase									=			self.get_extended_attention_mask(UpperCAmelCase       ,    UpperCAmelCase    )

							for lyr in self.encoders:
													_UpperCAmelCase									=			lyr(UpperCAmelCase       ,    UpperCAmelCase    )[0]
							_UpperCAmelCase									=			self.layer_norm(UpperCAmelCase    )

							return self.dropout_post(UpperCAmelCase    ), encoder_inputs_mask








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a      	=  logging.get_logger(__name__)

_a      	=  {
    '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/config.json''',
    '''distilbert-base-uncased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/config.json''',
    '''distilbert-base-cased-distilled-squad''': (
        '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json'''
    ),
    '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json''',
    '''distilbert-base-multilingual-cased''': (
        '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json'''
    ),
    '''distilbert-base-uncased-finetuned-sst-2-english''': (
        '''https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json'''
    ),
}





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=      "distilbert"
	UpperCamelCase__								=      {
	    "hidden_size": "dim",
	    "num_attention_heads": "n_heads",
	    "num_hidden_layers": "n_layers",
	}

	def __init__(							self       ,    UpperCAmelCase=3_0522       ,    UpperCAmelCase=512       ,    UpperCAmelCase=False       ,    UpperCAmelCase=6       ,    UpperCAmelCase=12       ,    UpperCAmelCase=768       ,    UpperCAmelCase=4 * 768       ,    UpperCAmelCase=0.1       ,    UpperCAmelCase=0.1       ,    UpperCAmelCase="gelu"       ,    UpperCAmelCase=0.02       ,    UpperCAmelCase=0.1       ,    UpperCAmelCase=0.2       ,    UpperCAmelCase=0       ,    **UpperCAmelCase       ,    ):
							"""simple docstring"""







							_UpperCAmelCase									=			vocab_size
							_UpperCAmelCase									=			max_position_embeddings
							_UpperCAmelCase									=			sinusoidal_pos_embds
							_UpperCAmelCase									=			n_layers
							_UpperCAmelCase									=			n_heads
							_UpperCAmelCase									=			dim
							_UpperCAmelCase									=			hidden_dim
							_UpperCAmelCase									=			dropout
							_UpperCAmelCase									=			attention_dropout
							_UpperCAmelCase									=			activation
							_UpperCAmelCase									=			initializer_range
							_UpperCAmelCase									=			qa_dropout
							_UpperCAmelCase									=			seq_classif_dropout
							super().__init__(**UpperCAmelCase       ,    pad_token_id=UpperCAmelCase    )





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	@property
	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							if self.task == "multiple-choice":
													_UpperCAmelCase									=			{0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
													_UpperCAmelCase									=			{0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]    )
