'''simple docstring'''


import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


snake_case__	:     List[Any]      =    logging.get_logger(__name__)

snake_case__	:     Dict      =    {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case__	:     List[str]      =    {
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

snake_case__	:     Optional[Any]      =    {
    '''microsoft/deberta-base''': 512,
    '''microsoft/deberta-large''': 512,
    '''microsoft/deberta-xlarge''': 512,
    '''microsoft/deberta-base-mnli''': 512,
    '''microsoft/deberta-large-mnli''': 512,
    '''microsoft/deberta-xlarge-mnli''': 512,
}

snake_case__	:     Any      =    {
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}






class 	__SCREAMING_SNAKE_CASE		(			lowerCamelCase_			):







  '''simple docstring'''






  lowerCamelCase_						:Union[str, Any]		       =						VOCAB_FILES_NAMES
  lowerCamelCase_						:str		       =						PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase_						:List[str]		       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase_						:Union[str, Any]		       =						['''input_ids''', '''attention_mask''', '''token_type_ids''']
  lowerCamelCase_						:Dict		       =						DebertaTokenizer


  def __init__(					self ,						snake_case_=None ,						snake_case_=None ,						snake_case_=None ,						snake_case_="replace" ,						snake_case_="[CLS]" ,						snake_case_="[SEP]" ,						snake_case_="[SEP]" ,						snake_case_="[CLS]" ,						snake_case_="[UNK]" ,						snake_case_="[PAD]" ,						snake_case_="[MASK]" ,						snake_case_=False ,						**snake_case_ ,						):




       '''simple docstring'''

       super().__init__(
           snake_case_ ,						snake_case_ ,						tokenizer_file=snake_case_ ,						errors=snake_case_ ,						bos_token=snake_case_ ,						eos_token=snake_case_ ,						unk_token=snake_case_ ,						sep_token=snake_case_ ,						cls_token=snake_case_ ,						pad_token=snake_case_ ,						mask_token=snake_case_ ,						add_prefix_space=snake_case_ ,						**snake_case_ ,						)
       UpperCAmelCase_				:  Optional[Any]  = kwargs.pop('add_bos_token' ,						snake_case_		)

       UpperCAmelCase_				:  Optional[int]  = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
       if pre_tok_state.get('add_prefix_space' ,						snake_case_		) != add_prefix_space:
            UpperCAmelCase_				:  List[Any]  = getattr(snake_case_ ,						pre_tok_state.pop('type'		)		)
            UpperCAmelCase_				:  Any  = add_prefix_space
            UpperCAmelCase_				:  Union[str, Any]  = pre_tok_class(**snake_case_		)

       UpperCAmelCase_				:  List[str]  = add_prefix_space


  @property
  def _UpperCamelCase	(					self		):




       '''simple docstring'''

       if self._mask_token is None:
            if self.verbose:
                 logger.error('Using mask_token, but it is not set yet.'		)
            return None
       return str(self._mask_token		)


  @mask_token.setter
  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  Union[str, Any]  = AddedToken(snake_case_ ,						lstrip=snake_case_ ,						rstrip=snake_case_		) if isinstance(snake_case_ ,						snake_case_		) else value
       UpperCAmelCase_				:  Tuple  = value


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       UpperCAmelCase_				:  Optional[int]  = [self.cls_token_id]
       UpperCAmelCase_				:  Optional[int]  = [self.sep_token_id]
       return cls + token_ids_a + sep + token_ids_a + sep


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = [self.sep_token_id]
       UpperCAmelCase_				:  str  = [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


  def _UpperCamelCase	(					self ,						*snake_case_ ,						**snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = kwargs.get('is_split_into_words' ,						snake_case_		)
       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*snake_case_ ,						**snake_case_		)


  def _UpperCamelCase	(					self ,						*snake_case_ ,						**snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[Any]  = kwargs.get('is_split_into_words' ,						snake_case_		)

       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*snake_case_ ,						**snake_case_		)


  def _UpperCamelCase	(					self ,						snake_case_ ,						snake_case_ = None		):




       '''simple docstring'''

       UpperCAmelCase_				:  str  = self._tokenizer.model.save(snake_case_ ,						name=snake_case_		)
       return tuple(snake_case_		)






  def _UpperCamelCase	(					self ,						snake_case_		):




       '''simple docstring'''

       UpperCAmelCase_				:  List[str]  = []
       for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(snake_case_ ,						add_special_tokens=snake_case_		) + [self.eos_token_id]		)

       if len(snake_case_		) > self.model_max_length:
            UpperCAmelCase_				:  int  = input_ids[-self.model_max_length :]
       return input_ids










import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase			:					Union[str, Any]         =							logging.get_logger(__name__)

UpperCamelCase			:					List[Any]         =							{
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       Tuple								=			"hubert"
					def __init__(						self       :						List[Any]	,      _lowerCAmelCase       :						Any=3_2	,      _lowerCAmelCase       :						Optional[Any]=7_6_8	,      _lowerCAmelCase       :						str=1_2	,      _lowerCAmelCase       :						List[Any]=1_2	,      _lowerCAmelCase       :						Optional[int]=3_0_7_2	,      _lowerCAmelCase       :						int="gelu"	,      _lowerCAmelCase       :						List[str]=0.1	,      _lowerCAmelCase       :						Union[str, Any]=0.1	,      _lowerCAmelCase       :						Optional[Any]=0.1	,      _lowerCAmelCase       :						Optional[int]=True	,      _lowerCAmelCase       :						Any=0.0	,      _lowerCAmelCase       :						Tuple=0.1	,      _lowerCAmelCase       :						Optional[int]=0.1	,      _lowerCAmelCase       :						str=0.02	,      _lowerCAmelCase       :						str=1E-5	,      _lowerCAmelCase       :						Dict="group"	,      _lowerCAmelCase       :						Tuple="gelu"	,      _lowerCAmelCase       :						List[str]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)	,      _lowerCAmelCase       :						List[str]=(5, 2, 2, 2, 2, 2, 2)	,      _lowerCAmelCase       :						Union[str, Any]=(1_0, 3, 3, 3, 3, 2, 2)	,      _lowerCAmelCase       :						Any=False	,      _lowerCAmelCase       :						Union[str, Any]=1_2_8	,      _lowerCAmelCase       :						Any=1_6	,      _lowerCAmelCase       :						Tuple=False	,      _lowerCAmelCase       :						Dict=True	,      _lowerCAmelCase       :						Union[str, Any]=0.05	,      _lowerCAmelCase       :						Optional[int]=1_0	,      _lowerCAmelCase       :						str=2	,      _lowerCAmelCase       :						Any=0.0	,      _lowerCAmelCase       :						Tuple=1_0	,      _lowerCAmelCase       :						int=0	,      _lowerCAmelCase       :						Tuple="sum"	,      _lowerCAmelCase       :						Tuple=False	,      _lowerCAmelCase       :						str=False	,      _lowerCAmelCase       :						Tuple=2_5_6	,      _lowerCAmelCase       :						List[str]=0	,      _lowerCAmelCase       :						int=1	,      _lowerCAmelCase       :						int=2	,      **_lowerCAmelCase       :						Optional[Any]	,      )							->					Dict:



											'''simple docstring'''





											super().__init__(**_lowerCAmelCase	,      pad_token_id=_lowerCAmelCase	,      bos_token_id=_lowerCAmelCase	,      eos_token_id=_lowerCAmelCase  )
											__magic_name__					=      hidden_size
											__magic_name__					=      feat_extract_norm
											__magic_name__					=      feat_extract_activation
											__magic_name__					=      list(_lowerCAmelCase  )
											__magic_name__					=      list(_lowerCAmelCase  )
											__magic_name__					=      list(_lowerCAmelCase  )
											__magic_name__					=      conv_bias
											__magic_name__					=      num_conv_pos_embeddings
											__magic_name__					=      num_conv_pos_embedding_groups
											__magic_name__					=      len(self.conv_dim  )
											__magic_name__					=      num_hidden_layers
											__magic_name__					=      intermediate_size
											__magic_name__					=      hidden_act
											__magic_name__					=      num_attention_heads
											__magic_name__					=      hidden_dropout
											__magic_name__					=      attention_dropout
											__magic_name__					=      activation_dropout
											__magic_name__					=      feat_proj_layer_norm
											__magic_name__					=      feat_proj_dropout
											__magic_name__					=      final_dropout
											__magic_name__					=      layerdrop
											__magic_name__					=      layer_norm_eps
											__magic_name__					=      initializer_range
											__magic_name__					=      vocab_size
											__magic_name__					=      do_stable_layer_norm
											__magic_name__					=      use_weighted_layer_sum
											__magic_name__					=      classifier_proj_size

											if (
											    (len(self.conv_stride  ) != self.num_feat_extract_layers)
											    or (len(self.conv_kernel  ) != self.num_feat_extract_layers)
											    or (len(self.conv_dim  ) != self.num_feat_extract_layers)
											):
																	raise ValueError(
																	    'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
																	    ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
																	    F''' {len(self.conv_dim  )}`, `len(config.conv_stride) = {len(self.conv_stride  )}`,'''
																	    F''' `len(config.conv_kernel) = {len(self.conv_kernel  )}`.'''  )

											# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
											__magic_name__					=      apply_spec_augment
											__magic_name__					=      mask_time_prob
											__magic_name__					=      mask_time_length
											__magic_name__					=      mask_time_min_masks
											__magic_name__					=      mask_feature_prob
											__magic_name__					=      mask_feature_length
											__magic_name__					=      mask_feature_min_masks

											# ctc loss
											__magic_name__					=      ctc_loss_reduction
											__magic_name__					=      ctc_zero_infinity







					@property
					def 	a			(						self       :						Optional[int]  )							->					Any:



											'''simple docstring'''





											return functools.reduce(operator.mul	,      self.conv_stride	,      1  )

