





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
     import torch

     from transformers import XLMRobertaModel



@require_sentencepiece
@require_tokenizers
@require_torch
class    _SCREAMING_SNAKE_CASE						(	unittest.TestCase    ):







 '''simple docstring'''

 @slow
 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      List[str]			  =XLMRobertaModel.from_pretrained("xlm-roberta-base")
   lowerCamelCase__:      int			  =torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]])
   # The dog is cute and lives in the garden house

   lowerCamelCase__:      Optional[Any]			  =torch.Size((1, 12, 768))  # batch_size, sequence_length, embedding_vector_dim
   lowerCamelCase__:      str			  =torch.tensor(
       [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]])
   #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
   #  xlmr.eval()
   #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
   with torch.no_grad():
     lowerCamelCase__:      Union[str, Any]			  =model(UpperCAmelCase_)["last_hidden_state"].detach()
   self.assertEqual(output.shape       , UpperCAmelCase_)
   # compare the actual values for a slice of last dim
   self.assertTrue(torch.allclose(output[:, :, -1]       , UpperCAmelCase_       , atol=1E-3))







 @slow
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Dict:



   '''simple docstring'''



   lowerCamelCase__:      Union[str, Any]			  =XLMRobertaModel.from_pretrained("xlm-roberta-large")
   lowerCamelCase__:      Tuple			  =torch.tensor([[0, 581, 10_269, 83, 99_942, 136, 60_742, 23, 70, 80_583, 18_276, 2]])
   # The dog is cute and lives in the garden house

   lowerCamelCase__:      Union[str, Any]			  =torch.Size((1, 12, 1_024))  # batch_size, sequence_length, embedding_vector_dim
   lowerCamelCase__:      Union[str, Any]			  =torch.tensor(
       [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]])
   #  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
   #  xlmr.eval()
   #  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
   with torch.no_grad():
     lowerCamelCase__:      List[Any]			  =model(UpperCAmelCase_)["last_hidden_state"].detach()
   self.assertEqual(output.shape       , UpperCAmelCase_)
   # compare the actual values for a slice of last dim
   self.assertTrue(torch.allclose(output[:, :, -1]       , UpperCAmelCase_       , atol=1E-3))








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A									=				logging.get_logger(__name__)

__A									=				{
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"unispeech"

 def __init__(self				:   Any       , UpperCAmelCase_				:   Any=32       , UpperCAmelCase_				:   List[str]=768       , UpperCAmelCase_				:   Any=12       , UpperCAmelCase_				:   Union[str, Any]=12       , UpperCAmelCase_				:   Optional[Any]=3_072       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Any=0.0       , UpperCAmelCase_				:   str=0.0       , UpperCAmelCase_				:   Dict=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   Optional[Any]=0.02       , UpperCAmelCase_				:   Union[str, Any]=1E-5       , UpperCAmelCase_				:   str="group"       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   Tuple=(512, 512, 512, 512, 512, 512, 512)       , UpperCAmelCase_				:   str=(5, 2, 2, 2, 2, 2, 2)       , UpperCAmelCase_				:   Any=(10, 3, 3, 3, 3, 2, 2)       , UpperCAmelCase_				:   Optional[Any]=False       , UpperCAmelCase_				:   str=128       , UpperCAmelCase_				:   int=16       , UpperCAmelCase_				:   Dict=False       , UpperCAmelCase_				:   Optional[int]=True       , UpperCAmelCase_				:   Dict=0.05       , UpperCAmelCase_				:   Optional[int]=10       , UpperCAmelCase_				:   Tuple=2       , UpperCAmelCase_				:   Union[str, Any]=0.0       , UpperCAmelCase_				:   int=10       , UpperCAmelCase_				:   List[Any]=0       , UpperCAmelCase_				:   Optional[Any]=320       , UpperCAmelCase_				:   int=2       , UpperCAmelCase_				:   Union[str, Any]=0.1       , UpperCAmelCase_				:   str=100       , UpperCAmelCase_				:   Any=256       , UpperCAmelCase_				:   int=256       , UpperCAmelCase_				:   Optional[Any]=0.1       , UpperCAmelCase_				:   str="mean"       , UpperCAmelCase_				:   Union[str, Any]=False       , UpperCAmelCase_				:   List[str]=False       , UpperCAmelCase_				:   List[Any]=256       , UpperCAmelCase_				:   Optional[int]=80       , UpperCAmelCase_				:   Optional[int]=0       , UpperCAmelCase_				:   Optional[Any]=1       , UpperCAmelCase_				:   Union[str, Any]=2       , UpperCAmelCase_				:   Dict=0.5       , **UpperCAmelCase_				:   Optional[int]       , )       ->str:



   '''simple docstring'''



   super().__init__(**UpperCAmelCase_       , pad_token_id=UpperCAmelCase_       , bos_token_id=UpperCAmelCase_       , eos_token_id=UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =hidden_size
   lowerCamelCase__:      List[str]			  =feat_extract_norm
   lowerCamelCase__:      Dict			  =feat_extract_activation
   lowerCamelCase__:      Optional[Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Any			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Dict			  =conv_bias
   lowerCamelCase__:      Optional[Any]			  =num_conv_pos_embeddings
   lowerCamelCase__:      Dict			  =num_conv_pos_embedding_groups
   lowerCamelCase__:      int			  =len(self.conv_dim)
   lowerCamelCase__:      Union[str, Any]			  =num_hidden_layers
   lowerCamelCase__:      Union[str, Any]			  =intermediate_size
   lowerCamelCase__:      Dict			  =hidden_act
   lowerCamelCase__:      List[Any]			  =num_attention_heads
   lowerCamelCase__:      Dict			  =hidden_dropout
   lowerCamelCase__:      Optional[Any]			  =attention_dropout
   lowerCamelCase__:      Optional[Any]			  =activation_dropout
   lowerCamelCase__:      Tuple			  =feat_proj_dropout
   lowerCamelCase__:      int			  =final_dropout
   lowerCamelCase__:      Optional[Any]			  =layerdrop
   lowerCamelCase__:      Dict			  =layer_norm_eps
   lowerCamelCase__:      Optional[Any]			  =initializer_range
   lowerCamelCase__:      int			  =num_ctc_classes
   lowerCamelCase__:      Tuple			  =vocab_size
   lowerCamelCase__:      Dict			  =do_stable_layer_norm
   lowerCamelCase__:      List[Any]			  =use_weighted_layer_sum
   lowerCamelCase__:      Dict			  =classifier_proj_size

   if (
       (len(self.conv_stride) != self.num_feat_extract_layers)
       or (len(self.conv_kernel) != self.num_feat_extract_layers)
       or (len(self.conv_dim) != self.num_feat_extract_layers)
   ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         F""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
         F""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   lowerCamelCase__:      int			  =apply_spec_augment
   lowerCamelCase__:      List[str]			  =mask_time_prob
   lowerCamelCase__:      Union[str, Any]			  =mask_time_length
   lowerCamelCase__:      List[Any]			  =mask_time_min_masks
   lowerCamelCase__:      Any			  =mask_feature_prob
   lowerCamelCase__:      Optional[Any]			  =mask_feature_length
   lowerCamelCase__:      List[str]			  =mask_feature_min_masks

   # parameters for pretraining with codevector quantized representations
   lowerCamelCase__:      Optional[Any]			  =num_codevectors_per_group
   lowerCamelCase__:      str			  =num_codevector_groups
   lowerCamelCase__:      Tuple			  =contrastive_logits_temperature
   lowerCamelCase__:      int			  =feat_quantizer_dropout
   lowerCamelCase__:      Any			  =num_negatives
   lowerCamelCase__:      List[str]			  =codevector_dim
   lowerCamelCase__:      Union[str, Any]			  =proj_codevector_dim
   lowerCamelCase__:      Any			  =diversity_loss_weight

   # ctc loss
   lowerCamelCase__:      Any			  =ctc_loss_reduction
   lowerCamelCase__:      Dict			  =ctc_zero_infinity

   # pretraining loss
   lowerCamelCase__:      Dict			  =replace_prob







 @property
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   return functools.reduce(operator.mul       , self.conv_stride       , 1)

