import math
import unittest

from transformers import BioGptConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        BioGptForCausalLM,
        BioGptForSequenceClassification,
        BioGptForTokenClassification,
        BioGptModel,
        BioGptTokenizer,
    )
    from transformers.models.biogpt.modeling_biogpt import BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST


class 	lowerCamelCase_     :






    '''simple docstring'''






    def __init__(	self , __lowercase , __lowercase=13 , __lowercase=7 , __lowercase=True , __lowercase=True , __lowercase=False , __lowercase=True , __lowercase=99 , __lowercase=32 , __lowercase=5 , __lowercase=4 , __lowercase=37 , __lowercase="gelu" , __lowercase=0.1 , __lowercase=0.1 , __lowercase=512 , __lowercase=16 , __lowercase=2 , __lowercase=0.02 , __lowercase=3 , __lowercase=4 , __lowercase=None , )       ->	Optional[Any]:
        __UpperCamelCase     :Tuple			  =		parent
        __UpperCamelCase     :Union[str, Any]			  =		batch_size
        __UpperCamelCase     :int			  =		seq_length
        __UpperCamelCase     :Optional[Any]			  =		is_training
        __UpperCamelCase     :Any			  =		use_input_mask
        __UpperCamelCase     :Any			  =		use_token_type_ids
        __UpperCamelCase     :str			  =		use_labels
        __UpperCamelCase     :int			  =		vocab_size
        __UpperCamelCase     :Optional[int]			  =		hidden_size
        __UpperCamelCase     :str			  =		num_hidden_layers
        __UpperCamelCase     :Union[str, Any]			  =		num_attention_heads
        __UpperCamelCase     :Dict			  =		intermediate_size
        __UpperCamelCase     :Tuple			  =		hidden_act
        __UpperCamelCase     :Dict			  =		hidden_dropout_prob
        __UpperCamelCase     :str			  =		attention_probs_dropout_prob
        __UpperCamelCase     :Union[str, Any]			  =		max_position_embeddings
        __UpperCamelCase     :Dict			  =		type_vocab_size
        __UpperCamelCase     :Optional[int]			  =		type_sequence_label_size
        __UpperCamelCase     :Tuple			  =		initializer_range
        __UpperCamelCase     :Optional[Any]			  =		num_labels
        __UpperCamelCase     :Any			  =		num_choices
        __UpperCamelCase     :Dict			  =		scope







    def 			UpperCamelCase__   (	self)       ->	Optional[int]:
        __UpperCamelCase     :Union[str, Any]			  =		ids_tensor([self.batch_size, self.seq_length] , self.vocab_size)

        __UpperCamelCase     :Union[str, Any]			  =		None
        if self.use_input_mask:
            __UpperCamelCase     :Dict			  =		random_attention_mask([self.batch_size, self.seq_length])

        __UpperCamelCase     :str			  =		None
        if self.use_token_type_ids:
            __UpperCamelCase     :Union[str, Any]			  =		ids_tensor([self.batch_size, self.seq_length] , self.type_vocab_size)

        __UpperCamelCase     :List[str]			  =		None
        __UpperCamelCase     :Any			  =		None
        __UpperCamelCase     :str			  =		None
        if self.use_labels:
            __UpperCamelCase     :int			  =		ids_tensor([self.batch_size] , self.type_sequence_label_size)
            __UpperCamelCase     :Any			  =		ids_tensor([self.batch_size, self.seq_length] , self.num_labels)
            __UpperCamelCase     :str			  =		ids_tensor([self.batch_size] , self.num_choices)

        __UpperCamelCase     :str			  =		self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







    def 			UpperCamelCase__   (	self)       ->	Union[str, Any]:
        return BioGptConfig(
            vocab_size=self.vocab_size , hidden_size=self.hidden_size , num_hidden_layers=self.num_hidden_layers , num_attention_heads=self.num_attention_heads , intermediate_size=self.intermediate_size , hidden_act=self.hidden_act , hidden_dropout_prob=self.hidden_dropout_prob , attention_probs_dropout_prob=self.attention_probs_dropout_prob , max_position_embeddings=self.max_position_embeddings , type_vocab_size=self.type_vocab_size , is_decoder=__lowercase , initializer_range=self.initializer_range , )







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase)       ->	List[Any]:
        __UpperCamelCase     :Any			  =		BioGptModel(config=__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase     :int			  =		model(__lowercase , attention_mask=__lowercase)
        __UpperCamelCase     :int			  =		model(__lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape , (self.batch_size, self.seq_length, self.hidden_size))







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , )       ->	str:
        __UpperCamelCase     :Dict			  =		BioGptForCausalLM(config=__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase     :Any			  =		model(__lowercase , attention_mask=__lowercase , token_type_ids=__lowercase , labels=__lowercase)
        self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.vocab_size))







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , *__lowercase)       ->	str:
        __UpperCamelCase     :Optional[Any]			  =		BioGptModel(config=__lowercase)
        model.to(__lowercase)
        model.eval()

        # create attention mask
        __UpperCamelCase     :Tuple			  =		torch.ones(input_ids.shape , dtype=torch.long , device=__lowercase)
        __UpperCamelCase     :Optional[Any]			  =		self.seq_length // 2
        __UpperCamelCase     :Tuple			  =		0

        # first forward pass
        __UpperCamelCase  ,       __UpperCamelCase     :Optional[int]			  =		model(__lowercase , attention_mask=__lowercase).to_tuple()

        # create hypothetical next token and extent to next_input_ids
        __UpperCamelCase     :List[Any]			  =		ids_tensor((self.batch_size, 1) , config.vocab_size)

        # change a random masked slice from input_ids
        __UpperCamelCase     :Union[str, Any]			  =		ids_tensor((1,) , __lowercase).item() + 1
        __UpperCamelCase     :str			  =		ids_tensor((self.batch_size, 1) , config.vocab_size).squeeze(-1)
        __UpperCamelCase     :int			  =		random_other_next_tokens

        # append to next input_ids and attn_mask
        __UpperCamelCase     :Tuple			  =		torch.cat([input_ids, next_tokens] , dim=-1)
        __UpperCamelCase     :List[str]			  =		torch.cat(
            [attn_mask, torch.ones((attn_mask.shape[0], 1) , dtype=torch.long , device=__lowercase)] , dim=1 , )

        # get two different outputs
        __UpperCamelCase     :List[str]			  =		model(__lowercase , attention_mask=__lowercase)['''last_hidden_state''']
        __UpperCamelCase     :Optional[int]			  =		model(__lowercase , past_key_values=__lowercase , attention_mask=__lowercase)['''last_hidden_state''']

        # select random slice
        __UpperCamelCase     :str			  =		ids_tensor((1,) , output_from_past.shape[-1]).item()
        __UpperCamelCase     :str			  =		output_from_no_past[:, -1, random_slice_idx].detach()
        __UpperCamelCase     :Tuple			  =		output_from_past[:, 0, random_slice_idx].detach()

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(__lowercase , __lowercase , atol=1E-3))







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , *__lowercase)       ->	List[str]:
        __UpperCamelCase     :str			  =		BioGptModel(config=__lowercase).to(__lowercase).eval()

        __UpperCamelCase     :Optional[Any]			  =		torch.ones(input_ids.shape , dtype=torch.long , device=__lowercase)

        # first forward pass
        __UpperCamelCase     :Dict			  =		model(__lowercase , attention_mask=__lowercase , use_cache=__lowercase)

        __UpperCamelCase  ,       __UpperCamelCase     :Optional[int]			  =		outputs.to_tuple()

        # create hypothetical multiple next token and extent to next_input_ids
        __UpperCamelCase     :Optional[Any]			  =		ids_tensor((self.batch_size, 3) , config.vocab_size)
        __UpperCamelCase     :List[str]			  =		ids_tensor((self.batch_size, 3) , 2)

        # append to next input_ids and
        __UpperCamelCase     :Optional[Any]			  =		torch.cat([input_ids, next_tokens] , dim=-1)
        __UpperCamelCase     :Union[str, Any]			  =		torch.cat([attention_mask, next_attn_mask] , dim=-1)

        __UpperCamelCase     :Tuple			  =		model(__lowercase , attention_mask=__lowercase)['''last_hidden_state''']
        __UpperCamelCase     :Optional[Any]			  =		model(__lowercase , attention_mask=__lowercase , past_key_values=__lowercase)[
            '''last_hidden_state'''
        ]

        # select random slice
        __UpperCamelCase     :Tuple			  =		ids_tensor((1,) , output_from_past.shape[-1]).item()
        __UpperCamelCase     :int			  =		output_from_no_past[:, -3:, random_slice_idx].detach()
        __UpperCamelCase     :Optional[int]			  =		output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(__lowercase , __lowercase , atol=1E-3))







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , *__lowercase , __lowercase=False)       ->	List[str]:
        __UpperCamelCase     :List[Any]			  =		BioGptForCausalLM(__lowercase)
        model.to(__lowercase)
        if gradient_checkpointing:
            model.gradient_checkpointing_enable()

        __UpperCamelCase     :str			  =		model(__lowercase , labels=__lowercase)
        self.parent.assertEqual(result.loss.shape , ())
        self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.vocab_size))
        result.loss.backward()







    def 			UpperCamelCase__   (	self , __lowercase , *__lowercase)       ->	int:
        __UpperCamelCase     :List[str]			  =		BioGptModel(__lowercase)
        __UpperCamelCase     :int			  =		model.config.initializer_range / math.sqrt(2 * model.config.num_hidden_layers)
        for key in model.state_dict().keys():
            if "c_proj" in key and "weight" in key:
                self.parent.assertLessEqual(abs(torch.std(model.state_dict()[key]) - model_std) , 0.0_01)
                self.parent.assertLessEqual(abs(torch.mean(model.state_dict()[key]) - 0.0) , 0.01)







    def 			UpperCamelCase__   (	self , __lowercase , __lowercase , __lowercase , __lowercase , __lowercase , *__lowercase)       ->	Dict:
        __UpperCamelCase     :List[str]			  =		self.num_labels
        __UpperCamelCase     :List[Any]			  =		BioGptForTokenClassification(__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase     :List[str]			  =		model(__lowercase , attention_mask=__lowercase , token_type_ids=__lowercase)
        self.parent.assertEqual(result.logits.shape , (self.batch_size, self.seq_length, self.num_labels))







    def 			UpperCamelCase__   (	self)       ->	List[str]:
        __UpperCamelCase     :List[Any]			  =		self.prepare_config_and_inputs()
        (
            (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       (
            __UpperCamelCase
        )  ,       
        )     :Union[str, Any]			  =		config_and_inputs
        __UpperCamelCase     :Any			  =		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
        return config, inputs_dict


@require_torch
class 	lowerCamelCase_     (   UpperCAmelCase_						,	UpperCAmelCase_						,	UpperCAmelCase_						,	unittest.TestCase ):






    '''simple docstring'''






    a__	:						Optional[int]						=      (
        (BioGptModel, BioGptForCausalLM, BioGptForSequenceClassification, BioGptForTokenClassification)
        if is_torch_available()
        else ()
    )
    a__	:						str						=      (BioGptForCausalLM,) if is_torch_available() else ()
    a__	:						str						=      (
        {
            """feature-extraction""": BioGptModel,
            """text-classification""": BioGptForSequenceClassification,
            """text-generation""": BioGptForCausalLM,
            """token-classification""": BioGptForTokenClassification,
            """zero-shot""": BioGptForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    a__	:						Any						=      False







    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        __UpperCamelCase     :Dict			  =		BioGptModelTester(self)
        __UpperCamelCase     :List[Any]			  =		ConfigTester(self , config_class=__lowercase , hidden_size=37)







    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        self.config_tester.run_common_tests()







    def 			UpperCamelCase__   (	self)       ->	str:
        __UpperCamelCase     :Tuple			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*__lowercase)







    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        __UpperCamelCase     :List[Any]			  =		self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
            __UpperCamelCase     :Optional[Any]			  =		type
            self.model_tester.create_and_check_model(*__lowercase)







    def 			UpperCamelCase__   (	self)       ->	Dict:
        __UpperCamelCase     :Tuple			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_biogpt_model_attention_mask_past(*__lowercase)







    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        __UpperCamelCase     :Tuple			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_forward_and_backwards(*__lowercase , gradient_checkpointing=__lowercase)







    def 			UpperCamelCase__   (	self)       ->	Optional[int]:
        __UpperCamelCase     :Optional[Any]			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_biogpt_model_past_large_inputs(*__lowercase)







    def 			UpperCamelCase__   (	self)       ->	Optional[int]:
        __UpperCamelCase     :int			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_biogpt_weight_initialization(*__lowercase)







    def 			UpperCamelCase__   (	self)       ->	Dict:
        __UpperCamelCase     :Dict			  =		self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_biogpt_for_token_classification(*__lowercase)







    @slow
    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        __UpperCamelCase     :Union[str, Any]			  =		BioGptForCausalLM.from_pretrained('''microsoft/biogpt''')
        model.to(__lowercase)
        __UpperCamelCase     :Dict			  =		BioGptTokenizer.from_pretrained('''microsoft/biogpt''')

        __UpperCamelCase     :Optional[int]			  =		'''left'''

        # Define PAD Token = EOS Token = 50256
        __UpperCamelCase     :List[str]			  =		tokenizer.eos_token
        __UpperCamelCase     :Dict			  =		model.config.eos_token_id

        # use different length sentences to test batching
        __UpperCamelCase     :str			  =		[
            '''Hello, my dog is a little''',
            '''Today, I''',
        ]

        __UpperCamelCase     :Optional[Any]			  =		tokenizer(__lowercase , return_tensors='''pt''' , padding=__lowercase)
        __UpperCamelCase     :str			  =		inputs['''input_ids'''].to(__lowercase)

        __UpperCamelCase     :Any			  =		model.generate(
            input_ids=__lowercase , attention_mask=inputs['''attention_mask'''].to(__lowercase) , )

        __UpperCamelCase     :str			  =		tokenizer(sentences[0] , return_tensors='''pt''').input_ids.to(__lowercase)
        __UpperCamelCase     :Union[str, Any]			  =		model.generate(input_ids=__lowercase)

        __UpperCamelCase     :Dict			  =		inputs_non_padded.shape[-1] - inputs['''attention_mask'''][-1].long().sum().cpu().item()
        __UpperCamelCase     :List[Any]			  =		tokenizer(sentences[1] , return_tensors='''pt''').input_ids.to(__lowercase)
        __UpperCamelCase     :Union[str, Any]			  =		model.generate(input_ids=__lowercase , max_length=model.config.max_length - num_paddings)

        __UpperCamelCase     :str			  =		tokenizer.batch_decode(__lowercase , skip_special_tokens=__lowercase)
        __UpperCamelCase     :str			  =		tokenizer.decode(output_non_padded[0] , skip_special_tokens=__lowercase)
        __UpperCamelCase     :int			  =		tokenizer.decode(output_padded[0] , skip_special_tokens=__lowercase)

        __UpperCamelCase     :Optional[Any]			  =		[
            '''Hello, my dog is a little bit bigger than a little bit.''',
            '''Today, I have a good idea of how to use the information''',
        ]
        self.assertListEqual(__lowercase , __lowercase)
        self.assertListEqual(__lowercase , [non_padded_sentence, padded_sentence])







    @slow
    def 			UpperCamelCase__   (	self)       ->	int:
        for model_name in BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __UpperCamelCase     :Optional[int]			  =		BioGptModel.from_pretrained(__lowercase)
            self.assertIsNotNone(__lowercase)







    def 			UpperCamelCase__   (	self)       ->	List[Any]:
        __UpperCamelCase  ,       __UpperCamelCase     :str			  =		self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCamelCase     :List[str]			  =		3
        __UpperCamelCase     :Optional[int]			  =		input_dict['''input_ids''']
        __UpperCamelCase     :Dict			  =		input_ids.ne(1).to(__lowercase)
        __UpperCamelCase     :Dict			  =		ids_tensor([self.model_tester.batch_size] , self.model_tester.type_sequence_label_size)
        __UpperCamelCase     :Optional[Any]			  =		BioGptForSequenceClassification(__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase     :Dict			  =		model(__lowercase , attention_mask=__lowercase , labels=__lowercase)
        self.assertEqual(result.logits.shape , (self.model_tester.batch_size, self.model_tester.num_labels))







    def 			UpperCamelCase__   (	self)       ->	List[str]:
        __UpperCamelCase  ,       __UpperCamelCase     :Any			  =		self.model_tester.prepare_config_and_inputs_for_common()
        __UpperCamelCase     :Tuple			  =		3
        __UpperCamelCase     :Optional[Any]			  =		'''multi_label_classification'''
        __UpperCamelCase     :Union[str, Any]			  =		input_dict['''input_ids''']
        __UpperCamelCase     :Optional[int]			  =		input_ids.ne(1).to(__lowercase)
        __UpperCamelCase     :Optional[Any]			  =		ids_tensor(
            [self.model_tester.batch_size, config.num_labels] , self.model_tester.type_sequence_label_size).to(torch.float)
        __UpperCamelCase     :Tuple			  =		BioGptForSequenceClassification(__lowercase)
        model.to(__lowercase)
        model.eval()
        __UpperCamelCase     :Union[str, Any]			  =		model(__lowercase , attention_mask=__lowercase , labels=__lowercase)
        self.assertEqual(result.logits.shape , (self.model_tester.batch_size, self.model_tester.num_labels))


@require_torch
class 	lowerCamelCase_     (   unittest.TestCase ):






    '''simple docstring'''






    @slow
    def 			UpperCamelCase__   (	self)       ->	str:
        __UpperCamelCase     :Dict			  =		BioGptForCausalLM.from_pretrained('''microsoft/biogpt''')
        __UpperCamelCase     :str			  =		torch.tensor([[2, 4_805, 9, 656, 21]])
        __UpperCamelCase     :int			  =		model(__lowercase)[0]

        __UpperCamelCase     :List[str]			  =		42_384

        __UpperCamelCase     :str			  =		torch.Size((1, 5, vocab_size))
        self.assertEqual(output.shape , __lowercase)

        __UpperCamelCase     :List[Any]			  =		torch.tensor(
            [[[-9.52_36, -9.89_18, 10.45_57], [-11.04_69, -9.64_23, 8.10_22], [-8.86_64, -7.88_26, 5.53_25]]])

        self.assertTrue(torch.allclose(output[:, :3, :3] , __lowercase , atol=1E-4))







    @slow
    def 			UpperCamelCase__   (	self)       ->	Tuple:
        __UpperCamelCase     :Tuple			  =		BioGptTokenizer.from_pretrained('''microsoft/biogpt''')
        __UpperCamelCase     :Optional[Any]			  =		BioGptForCausalLM.from_pretrained('''microsoft/biogpt''')
        model.to(__lowercase)

        torch.manual_seed(0)
        __UpperCamelCase     :List[str]			  =		tokenizer('''COVID-19 is''' , return_tensors='''pt''').to(__lowercase)
        __UpperCamelCase     :Optional[int]			  =		model.generate(
            **__lowercase , min_length=100 , max_length=1_024 , num_beams=5 , early_stopping=__lowercase , )
        __UpperCamelCase     :List[str]			  =		tokenizer.decode(output_ids[0] , skip_special_tokens=__lowercase)

        __UpperCamelCase     :str			  =		(
            '''COVID-19 is a global pandemic caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2), the'''
            ''' causative agent of coronavirus disease 2019 (COVID-19), which has spread to more than 200 countries and'''
            ''' territories, including the United States (US), Canada, Australia, New Zealand, the United Kingdom (UK),'''
            ''' and the United States of America (USA), as of March 11, 2020, with more than 800,000 confirmed cases and'''
            ''' more than 800,000 deaths.'''
        )
        self.assertEqual(__lowercase , __lowercase)






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase							 =					logging.get_logger(__name__)

__lowercase							 =					{
    '''facebook/wav2vec2-base-960h''': '''https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json''',
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    a__	:						Union[str, Any]						=      """wav2vec2"""







    def __init__(	self , __lowercase=32 , __lowercase=768 , __lowercase=12 , __lowercase=12 , __lowercase=3_072 , __lowercase="gelu" , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.0 , __lowercase=0.0 , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.02 , __lowercase=1E-5 , __lowercase="group" , __lowercase="gelu" , __lowercase=(512, 512, 512, 512, 512, 512, 512) , __lowercase=(5, 2, 2, 2, 2, 2, 2) , __lowercase=(10, 3, 3, 3, 3, 2, 2) , __lowercase=False , __lowercase=128 , __lowercase=16 , __lowercase=False , __lowercase=True , __lowercase=0.05 , __lowercase=10 , __lowercase=2 , __lowercase=0.0 , __lowercase=10 , __lowercase=0 , __lowercase=320 , __lowercase=2 , __lowercase=0.1 , __lowercase=100 , __lowercase=256 , __lowercase=256 , __lowercase=0.1 , __lowercase="sum" , __lowercase=False , __lowercase=False , __lowercase=256 , __lowercase=(512, 512, 512, 512, 1_500) , __lowercase=(5, 3, 3, 1, 1) , __lowercase=(1, 2, 3, 1, 1) , __lowercase=512 , __lowercase=0 , __lowercase=1 , __lowercase=2 , __lowercase=False , __lowercase=3 , __lowercase=2 , __lowercase=3 , __lowercase=None , __lowercase=None , **__lowercase , )       ->	int:
        super().__init__(**__lowercase , pad_token_id=__lowercase , bos_token_id=__lowercase , eos_token_id=__lowercase)
        __UpperCamelCase     :Any			  =		hidden_size
        __UpperCamelCase     :int			  =		feat_extract_norm
        __UpperCamelCase     :Tuple			  =		feat_extract_activation
        __UpperCamelCase     :Union[str, Any]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :int			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		conv_bias
        __UpperCamelCase     :Optional[int]			  =		num_conv_pos_embeddings
        __UpperCamelCase     :Dict			  =		num_conv_pos_embedding_groups
        __UpperCamelCase     :Any			  =		len(self.conv_dim)
        __UpperCamelCase     :List[str]			  =		num_hidden_layers
        __UpperCamelCase     :int			  =		intermediate_size
        __UpperCamelCase     :str			  =		hidden_act
        __UpperCamelCase     :Any			  =		num_attention_heads
        __UpperCamelCase     :int			  =		hidden_dropout
        __UpperCamelCase     :Tuple			  =		attention_dropout
        __UpperCamelCase     :List[str]			  =		activation_dropout
        __UpperCamelCase     :Optional[Any]			  =		feat_proj_dropout
        __UpperCamelCase     :Any			  =		final_dropout
        __UpperCamelCase     :Any			  =		layerdrop
        __UpperCamelCase     :str			  =		layer_norm_eps
        __UpperCamelCase     :Optional[Any]			  =		initializer_range
        __UpperCamelCase     :List[str]			  =		vocab_size
        __UpperCamelCase     :str			  =		do_stable_layer_norm
        __UpperCamelCase     :Union[str, Any]			  =		use_weighted_layer_sum

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                f""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
                f""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        __UpperCamelCase     :List[Any]			  =		apply_spec_augment
        __UpperCamelCase     :Tuple			  =		mask_time_prob
        __UpperCamelCase     :int			  =		mask_time_length
        __UpperCamelCase     :Dict			  =		mask_time_min_masks
        __UpperCamelCase     :str			  =		mask_feature_prob
        __UpperCamelCase     :List[str]			  =		mask_feature_length
        __UpperCamelCase     :Union[str, Any]			  =		mask_feature_min_masks

        # parameters for pretraining with codevector quantized representations
        __UpperCamelCase     :Optional[Any]			  =		num_codevectors_per_group
        __UpperCamelCase     :List[Any]			  =		num_codevector_groups
        __UpperCamelCase     :Tuple			  =		contrastive_logits_temperature
        __UpperCamelCase     :Optional[int]			  =		feat_quantizer_dropout
        __UpperCamelCase     :Optional[int]			  =		num_negatives
        __UpperCamelCase     :List[Any]			  =		codevector_dim
        __UpperCamelCase     :str			  =		proj_codevector_dim
        __UpperCamelCase     :List[str]			  =		diversity_loss_weight

        # ctc loss
        __UpperCamelCase     :Tuple			  =		ctc_loss_reduction
        __UpperCamelCase     :Tuple			  =		ctc_zero_infinity

        # adapter
        __UpperCamelCase     :List[str]			  =		add_adapter
        __UpperCamelCase     :Tuple			  =		adapter_kernel_size
        __UpperCamelCase     :str			  =		adapter_stride
        __UpperCamelCase     :Tuple			  =		num_adapter_layers
        __UpperCamelCase     :Tuple			  =		output_hidden_size or hidden_size
        __UpperCamelCase     :Optional[Any]			  =		adapter_attn_dim

        # SequenceClassification-specific parameter. Feel free to ignore for other classes.
        __UpperCamelCase     :Optional[Any]			  =		classifier_proj_size

        # XVector-specific parameters. Feel free to ignore for other classes.
        __UpperCamelCase     :Optional[int]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :str			  =		xvector_output_dim







    @property
    def 			UpperCamelCase__   (	self)       ->	List[str]:
        return functools.reduce(operator.mul , self.conv_stride , 1)





