import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
         RwkvForCausalLM,
         RwkvModel,
     )
     from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
     lowercase_  							=			False







class      SCREAMING_SNAKE_CASE__					:

       def __init__(							self					:     Union[str, Any]			,			_lowerCAmelCase					:     str			,			_lowerCAmelCase					:     str=14			,			_lowerCAmelCase					:     Tuple=7			,			_lowerCAmelCase					:     Dict=True			,			_lowerCAmelCase					:     int=False			,			_lowerCAmelCase					:     Optional[int]=True			,			_lowerCAmelCase					:     Any=True			,			_lowerCAmelCase					:     str=True			,			_lowerCAmelCase					:     List[Any]=99			,			_lowerCAmelCase					:     Optional[int]=32			,			_lowerCAmelCase					:     Union[str, Any]=5			,			_lowerCAmelCase					:     int=37			,			_lowerCAmelCase					:     Dict="gelu"			,			_lowerCAmelCase					:     int=0.1			,			_lowerCAmelCase					:     Optional[Any]=0.1			,			_lowerCAmelCase					:     Optional[Any]=5_12			,			_lowerCAmelCase					:     Tuple=16			,			_lowerCAmelCase					:     List[Any]=2			,			_lowerCAmelCase					:     Any=3			,			_lowerCAmelCase					:     str=4			,			_lowerCAmelCase					:     str=None			,			):
           __snake_case       :				Optional[int] 			=    parent
           __snake_case       :				int 			=    batch_size
           __snake_case       :				Any 			=    seq_length
           __snake_case       :				Tuple 			=    is_training
           __snake_case       :				Union[str, Any] 			=    use_token_type_ids
           __snake_case       :				List[str] 			=    use_input_mask
           __snake_case       :				Optional[int] 			=    use_labels
           __snake_case       :				List[Any] 			=    use_mc_token_ids
           __snake_case       :				Optional[Any] 			=    vocab_size
           __snake_case       :				List[Any] 			=    hidden_size
           __snake_case       :				Any 			=    num_hidden_layers
           __snake_case       :				Tuple 			=    intermediate_size
           __snake_case       :				Optional[Any] 			=    hidden_act
           __snake_case       :				List[str] 			=    hidden_dropout_prob
           __snake_case       :				Union[str, Any] 			=    attention_probs_dropout_prob
           __snake_case       :				List[Any] 			=    max_position_embeddings
           __snake_case       :				List[Any] 			=    type_vocab_size
           __snake_case       :				Tuple 			=    type_sequence_label_size
           __snake_case       :				List[Any] 			=    num_labels
           __snake_case       :				List[str] 			=    num_choices
           __snake_case       :				List[str] 			=    scope
           __snake_case       :				Union[str, Any] 			=    vocab_size - 1
           __snake_case       :				str 			=    vocab_size - 1
           __snake_case       :				str 			=    vocab_size - 1

       def 			snake_case__    (							self					:     str			):
           return RwkvConfig.from_pretrained("""sgugger/rwkv-4-pile-7b"""			)

       def 			snake_case__    (							self					:     int			,			_lowerCAmelCase					:     Any=False			,			_lowerCAmelCase					:     List[str]=False			,			_lowerCAmelCase					:     str=False			):
           __snake_case       :				Any 			=    ids_tensor([self.batch_size, self.seq_length]			,			self.vocab_size			)

           __snake_case       :				List[Any] 			=    None
           if self.use_input_mask:
               __snake_case       :				Optional[Any] 			=    random_attention_mask([self.batch_size, self.seq_length]			)

           __snake_case       :				Union[str, Any] 			=    None
           if self.use_token_type_ids:
               __snake_case       :				List[Any] 			=    ids_tensor([self.batch_size, self.seq_length]			,			self.type_vocab_size			)

           __snake_case       :				int 			=    None
           if self.use_mc_token_ids:
               __snake_case       :				Optional[Any] 			=    ids_tensor([self.batch_size, self.num_choices]			,			self.seq_length			)

           __snake_case       :				Tuple 			=    None
           __snake_case       :				Optional[Any] 			=    None
           __snake_case       :				Optional[int] 			=    None
           if self.use_labels:
               __snake_case       :				int 			=    ids_tensor([self.batch_size]			,			self.type_sequence_label_size			)
               __snake_case       :				Tuple 			=    ids_tensor([self.batch_size, self.seq_length]			,			self.num_labels			)
               __snake_case       :				str 			=    ids_tensor([self.batch_size]			,			self.num_choices			)

           __snake_case       :				Dict 			=    self.get_config(
               gradient_checkpointing=_lowerCAmelCase			,			scale_attn_by_inverse_layer_idx=_lowerCAmelCase			,			reorder_and_upcast_attn=_lowerCAmelCase			,			)

           return (
               config,
               input_ids,
               input_mask,
               None,
               token_type_ids,
               mc_token_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )

       def 			snake_case__    (							self					:     int			,			_lowerCAmelCase					:     Any=False			,			_lowerCAmelCase					:     Optional[int]=False			,			_lowerCAmelCase					:     Optional[int]=False			):
           return RwkvConfig(
               vocab_size=self.vocab_size			,			hidden_size=self.hidden_size			,			num_hidden_layers=self.num_hidden_layers			,			intermediate_size=self.intermediate_size			,			activation_function=self.hidden_act			,			resid_pdrop=self.hidden_dropout_prob			,			attn_pdrop=self.attention_probs_dropout_prob			,			n_positions=self.max_position_embeddings			,			type_vocab_size=self.type_vocab_size			,			use_cache=_lowerCAmelCase			,			bos_token_id=self.bos_token_id			,			eos_token_id=self.eos_token_id			,			pad_token_id=self.pad_token_id			,			gradient_checkpointing=_lowerCAmelCase			,			scale_attn_by_inverse_layer_idx=_lowerCAmelCase			,			reorder_and_upcast_attn=_lowerCAmelCase			,			)

       def 			snake_case__    (							self					:     Tuple			):
           __snake_case       :				Optional[int] 			=    self.get_config()
           __snake_case       :				List[Any] 			=    3_00
           return config

       def 			snake_case__    (							self					:     Union[str, Any]			):
           (
               (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, 
           )       :				Dict 			=    self.prepare_config_and_inputs()

           __snake_case       :				Optional[Any] 			=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
           __snake_case       :				List[str] 			=    ids_tensor([self.batch_size, self.seq_length]			,			vocab_size=2			)

           return (
               config,
               input_ids,
               input_mask,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )

       def 			snake_case__    (							self					:     Tuple			,			_lowerCAmelCase					:     List[Any]			,			_lowerCAmelCase					:     Optional[Any]			,			_lowerCAmelCase					:     Dict			,			_lowerCAmelCase					:     Any			,			_lowerCAmelCase					:     Dict			,			*_lowerCAmelCase					:     List[Any]			):
           __snake_case       :				Dict 			=    True
           __snake_case       :				Optional[Any] 			=    RwkvModel(config=_lowerCAmelCase			)
           model.to(_lowerCAmelCase			)
           model.eval()

           __snake_case       :				Any 			=    model(_lowerCAmelCase			)

           self.parent.assertEqual(result.last_hidden_state.shape			,			(self.batch_size, self.seq_length, self.hidden_size)			)
           self.parent.assertEqual(len(result.hidden_states			)			,			config.num_hidden_layers + 1			)

       def 			snake_case__    (							self					:     Any			,			_lowerCAmelCase					:     Dict			,			_lowerCAmelCase					:     str			,			_lowerCAmelCase					:     List[Any]			,			_lowerCAmelCase					:     Union[str, Any]			,			_lowerCAmelCase					:     int			,			*_lowerCAmelCase					:     str			):
           __snake_case       :				int 			=    RwkvForCausalLM(_lowerCAmelCase			)
           model.to(_lowerCAmelCase			)
           model.eval()

           __snake_case       :				int 			=    model(_lowerCAmelCase			,			labels=_lowerCAmelCase			)
           self.parent.assertEqual(result.loss.shape			,			()			)
           self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.vocab_size)			)

       def 			snake_case__    (							self					:     str			,			_lowerCAmelCase					:     Optional[int]			,			_lowerCAmelCase					:     Tuple			,			_lowerCAmelCase					:     List[Any]			,			_lowerCAmelCase					:     List[str]			,			_lowerCAmelCase					:     Dict			,			*_lowerCAmelCase					:     Union[str, Any]			):
           __snake_case       :				Tuple 			=    RwkvModel(config=_lowerCAmelCase			)
           model.to(_lowerCAmelCase			)
           model.eval()

           __snake_case       :				Optional[int] 			=    model(_lowerCAmelCase			)
           __snake_case       :				List[str] 			=    outputs.last_hidden_state

           __snake_case       :				Any 			=    model(input_ids[:, :2]			)
           __snake_case       :				Optional[int] 			=    outputs.last_hidden_state

           # Using the state computed on the first inputs, we will get the same output
           __snake_case       :				Tuple 			=    model(input_ids[:, 2:]			,			state=outputs.state			)
           __snake_case       :				Optional[Any] 			=    outputs.last_hidden_state

           self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]			,			dim=1			)			,			_lowerCAmelCase			,			atol=1e-5			)			)

       def 			snake_case__    (							self					:     Union[str, Any]			,			_lowerCAmelCase					:     List[str]			,			_lowerCAmelCase					:     Dict			,			_lowerCAmelCase					:     List[str]			,			_lowerCAmelCase					:     int			,			_lowerCAmelCase					:     Optional[int]			,			*_lowerCAmelCase					:     Any			,			_lowerCAmelCase					:     Optional[int]=False			):
           __snake_case       :				Optional[Any] 			=    RwkvForCausalLM(_lowerCAmelCase			)
           model.to(_lowerCAmelCase			)
           if gradient_checkpointing:
               model.gradient_checkpointing_enable()

           __snake_case       :				str 			=    model(_lowerCAmelCase			,			labels=_lowerCAmelCase			)
           self.parent.assertEqual(result.loss.shape			,			()			)
           self.parent.assertEqual(result.logits.shape			,			(self.batch_size, self.seq_length, self.vocab_size)			)
           result.loss.backward()





       def 			snake_case__    (							self					:     int			):
           __snake_case       :				Dict 			=    self.prepare_config_and_inputs()

           (
               (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, (
               __snake_case
           )			, 
           )       :				List[Any] 			=    config_and_inputs

           __snake_case       :				str 			=    {"""input_ids""": input_ids}

           return config, inputs_dict







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0   ,							reason="See https://github.com/huggingface/transformers/pull/24204"    )
@require_torch
class      SCREAMING_SNAKE_CASE__					(  __UpperCamelCase   ,							__UpperCamelCase   ,							__UpperCamelCase   ,							unittest.TestCase    ):
       A							:	Optional[int]          =	(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
       A							:	Union[str, Any]          =	(
           {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {}
       )
       # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
       A							:	Any          =	False
       A							:	List[Any]          =	False
       A							:	Dict          =	False
       A							:	str          =	False
       A							:	Dict          =	False  # Rwkv does not support head masking

       def 			snake_case__    (							self					:     List[str]			):
           __snake_case       :				Optional[Any] 			=    RwkvModelTester(self			)
           __snake_case       :				str 			=    ConfigTester(
               self			,			config_class=_lowerCAmelCase			,			n_embd=37			,			common_properties=["""hidden_size""", """num_hidden_layers"""]			)

       def 			snake_case__    (							self					:     Any			,			_lowerCAmelCase					:     Union[str, Any]			,			_lowerCAmelCase					:     int			,			_lowerCAmelCase					:     str=None			):
           if isinstance(_lowerCAmelCase			,			torch.Tensor			):
               __snake_case			, __snake_case       :				Tuple 			=    member.max().item(), member.min().item()
           elif isinstance(_lowerCAmelCase			,			_lowerCAmelCase			) or isinstance(_lowerCAmelCase			,			_lowerCAmelCase			):
               __snake_case			, __snake_case       :				Optional[int] 			=    max(_lowerCAmelCase			), min(_lowerCAmelCase			)

           if not isinstance(_lowerCAmelCase			,			_lowerCAmelCase			):
               raise TypeError("""container should be a list or tuple"""			)
           elif len(_lowerCAmelCase			) != 2:
               raise ValueError("""container should have 2 elements"""			)

           __snake_case			, __snake_case       :				Tuple 			=    container

           __snake_case       :				Optional[Any] 			=    (min_value >= expected_min) and (max_value <= expected_max)

           if not is_inside_interval:
               __snake_case       :				Union[str, Any] 			=    """%s not found in %s""" % (safe_repr(_lowerCAmelCase			), safe_repr(_lowerCAmelCase			))
               self.fail(self._formatMessage(_lowerCAmelCase			,			_lowerCAmelCase			)			)

       def 			snake_case__    (							self					:     Tuple			):
           self.config_tester.run_common_tests()

       def 			snake_case__    (							self					:     Optional[Any]			):
           __snake_case       :				Optional[int] 			=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_rwkv_model(*_lowerCAmelCase			)

       def 			snake_case__    (							self					:     Union[str, Any]			):
           __snake_case       :				Any 			=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_causl_lm(*_lowerCAmelCase			)

       def 			snake_case__    (							self					:     List[Any]			):
           __snake_case       :				Optional[Any] 			=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_state_equivalency(*_lowerCAmelCase			)

       def 			snake_case__    (							self					:     List[str]			):
           __snake_case			, __snake_case       :				Optional[int] 			=    self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               __snake_case       :				Dict 			=    model_class(config=_lowerCAmelCase			)
               for name, param in model.named_parameters():
                   if "time_decay" in name:
                       if param.requires_grad:
                           self.assertTrue(param.data.max().item() == 3.0			)
                           self.assertTrue(param.data.min().item() == -5.0			)
                   elif "time_first" in name:
                       if param.requires_grad:
                           # check if it's a ones like
                           self.assertTrue(torch.allclose(param.data			,			torch.ones_like(param.data			)			,			atol=1e-5			,			rtol=1e-5			)			)
                   elif any(x in name for x in ["""time_mix_key""", """time_mix_receptance"""]			):
                       if param.requires_grad:
                           self.assertInterval(
                               param.data			,			[0.0, 1.0]			,			msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''			,			)
                   elif "time_mix_value" in name:
                       if param.requires_grad:
                           self.assertInterval(
                               param.data			,			[0.0, 1.3]			,			msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''			,			)

       def 			snake_case__    (							self					:     int			):
           __snake_case			, __snake_case       :				Optional[Any] 			=    self.model_tester.prepare_config_and_inputs_for_common()
           __snake_case       :				Any 			=    True

           __snake_case       :				List[Any] 			=    getattr(self.model_tester			,			"""seq_length"""			,			_lowerCAmelCase			)

           for model_class in self.all_model_classes:
               __snake_case       :				Any 			=    True
               __snake_case       :				Tuple 			=    False
               __snake_case       :				Any 			=    True
               __snake_case       :				Union[str, Any] 			=    model_class(_lowerCAmelCase			)
               model.to(_lowerCAmelCase			)
               model.eval()

               __snake_case       :				int 			=    self._prepare_for_class(_lowerCAmelCase			,			_lowerCAmelCase			)
               __snake_case       :				Dict 			=    inputs["""input_ids"""].shape[0]
               with torch.no_grad():
                   __snake_case       :				Optional[int] 			=    model(**_lowerCAmelCase			)
               __snake_case       :				str 			=    outputs.attentions
               self.assertEqual(len(_lowerCAmelCase			)			,			self.model_tester.num_hidden_layers			)

               # check that output_attentions also work using config
               del inputs_dict["output_attentions"]
               __snake_case       :				int 			=    True
               __snake_case       :				int 			=    model_class(_lowerCAmelCase			)
               model.to(_lowerCAmelCase			)
               model.eval()

               __snake_case       :				Tuple 			=    self._prepare_for_class(_lowerCAmelCase			,			_lowerCAmelCase			)
               __snake_case       :				Any 			=    inputs["""input_ids"""].shape[0]
               with torch.no_grad():
                   __snake_case       :				List[str] 			=    model(**_lowerCAmelCase			)
               __snake_case       :				Dict 			=    outputs.attentions
               self.assertEqual(len(_lowerCAmelCase			)			,			self.model_tester.num_hidden_layers			)

               self.assertListEqual(
                   list(attentions[0].shape[-3:]			)			,			[batch_size, seq_len, config.hidden_size]			,			)
               __snake_case       :				Dict 			=    len(_lowerCAmelCase			)

               # Check attention is always last and order is fine
               __snake_case       :				List[str] 			=    True
               __snake_case       :				List[str] 			=    True
               __snake_case       :				str 			=    model_class(_lowerCAmelCase			)
               model.to(_lowerCAmelCase			)
               model.eval()

               __snake_case       :				str 			=    self._prepare_for_class(_lowerCAmelCase			,			_lowerCAmelCase			)
               __snake_case       :				List[Any] 			=    inputs["""input_ids"""].shape[0]
               with torch.no_grad():
                   __snake_case       :				str 			=    model(**_lowerCAmelCase			)

               __snake_case       :				Optional[int] 			=    1
               self.assertEqual(out_len + added_hidden_states			,			len(_lowerCAmelCase			)			)

               __snake_case       :				str 			=    outputs.attentions

               self.assertEqual(len(_lowerCAmelCase			)			,			self.model_tester.num_hidden_layers			)
               self.assertListEqual(
                   list(self_attentions[0].shape[-3:]			)			,			[batch_size, seq_len, config.hidden_size]			,			)





       @slow
       def 			snake_case__    (							self					:     Optional[Any]			):
           for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               __snake_case       :				Dict 			=    RwkvModel.from_pretrained(_lowerCAmelCase			)
               self.assertIsNotNone(_lowerCAmelCase			)







@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0   ,							reason="See https://github.com/huggingface/transformers/pull/24204"    )
@slow
class      SCREAMING_SNAKE_CASE__					(  unittest.TestCase    ):

       def 			snake_case__    (							self					:     Any			):
           __snake_case       :				Optional[int] 			=    """RWKV/rwkv-4-169m-pile"""
           __snake_case       :				List[str] 			=    AutoTokenizer.from_pretrained(self.model_id			)

       def 			snake_case__    (							self					:     Optional[int]			):
           __snake_case       :				Any 			=    """Hello my name is Jasmine and I am a newbie to the"""
           __snake_case       :				Any 			=    RwkvForCausalLM.from_pretrained(self.model_id			).to(_lowerCAmelCase			)

           __snake_case       :				List[str] 			=    self.tokenizer("""Hello my name is"""			,			return_tensors="""pt"""			).input_ids.to(_lowerCAmelCase			)
           __snake_case       :				List[Any] 			=    model.generate(_lowerCAmelCase			,			max_new_tokens=10			)
           __snake_case       :				str 			=    self.tokenizer.decode(output[0].tolist()			)

           self.assertEqual(_lowerCAmelCase			,			_lowerCAmelCase			)





       def 			snake_case__    (							self					:     Optional[Any]			):
           __snake_case       :				str 			=    """Hello my name is Jasmine and I am a newbie to the"""

           __snake_case       :				Optional[Any] 			=    self.tokenizer("""Hello my name is"""			,			return_tensors="""pt"""			).input_ids.to(_lowerCAmelCase			)
           __snake_case       :				List[Any] 			=    RwkvForCausalLM.from_pretrained(self.model_id			,			torch_dtype=torch.bfloataa			).to(_lowerCAmelCase			)

           __snake_case       :				Any 			=    model.generate(_lowerCAmelCase			,			max_new_tokens=10			)
           __snake_case       :				List[Any] 			=    self.tokenizer.decode(output[0].tolist()			)

           self.assertEqual(_lowerCAmelCase			,			_lowerCAmelCase			)




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_  							=			logging.get_logger(__name__)

lowercase_  							=			{
    "facebook/hubert-base-ls960": "https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json",
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}







class      SCREAMING_SNAKE_CASE__					(  __UpperCamelCase    ):
       A							:	List[str]          =	"hubert"

       def __init__(							self					:     List[str]			,			_lowerCAmelCase					:     Union[str, Any]=32			,			_lowerCAmelCase					:     Any=7_68			,			_lowerCAmelCase					:     Optional[Any]=12			,			_lowerCAmelCase					:     Optional[int]=12			,			_lowerCAmelCase					:     int=30_72			,			_lowerCAmelCase					:     Optional[Any]="gelu"			,			_lowerCAmelCase					:     List[str]=0.1			,			_lowerCAmelCase					:     Any=0.1			,			_lowerCAmelCase					:     List[str]=0.1			,			_lowerCAmelCase					:     Dict=True			,			_lowerCAmelCase					:     Dict=0.0			,			_lowerCAmelCase					:     List[str]=0.1			,			_lowerCAmelCase					:     Optional[int]=0.1			,			_lowerCAmelCase					:     Dict=0.02			,			_lowerCAmelCase					:     Any=1e-5			,			_lowerCAmelCase					:     str="group"			,			_lowerCAmelCase					:     Any="gelu"			,			_lowerCAmelCase					:     List[str]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)			,			_lowerCAmelCase					:     Any=(5, 2, 2, 2, 2, 2, 2)			,			_lowerCAmelCase					:     Dict=(10, 3, 3, 3, 3, 2, 2)			,			_lowerCAmelCase					:     List[str]=False			,			_lowerCAmelCase					:     Optional[Any]=1_28			,			_lowerCAmelCase					:     int=16			,			_lowerCAmelCase					:     Tuple=False			,			_lowerCAmelCase					:     int=True			,			_lowerCAmelCase					:     int=0.05			,			_lowerCAmelCase					:     Any=10			,			_lowerCAmelCase					:     Dict=2			,			_lowerCAmelCase					:     Dict=0.0			,			_lowerCAmelCase					:     Optional[int]=10			,			_lowerCAmelCase					:     Union[str, Any]=0			,			_lowerCAmelCase					:     int="sum"			,			_lowerCAmelCase					:     Optional[int]=False			,			_lowerCAmelCase					:     str=False			,			_lowerCAmelCase					:     Optional[Any]=2_56			,			_lowerCAmelCase					:     Any=0			,			_lowerCAmelCase					:     List[Any]=1			,			_lowerCAmelCase					:     List[str]=2			,			**_lowerCAmelCase					:     Optional[int]			,			):
           super().__init__(**_lowerCAmelCase			,			pad_token_id=_lowerCAmelCase			,			bos_token_id=_lowerCAmelCase			,			eos_token_id=_lowerCAmelCase			)
           __snake_case       :				Optional[int] 			=    hidden_size
           __snake_case       :				Optional[Any] 			=    feat_extract_norm
           __snake_case       :				Optional[int] 			=    feat_extract_activation
           __snake_case       :				List[str] 			=    list(_lowerCAmelCase			)
           __snake_case       :				Any 			=    list(_lowerCAmelCase			)
           __snake_case       :				List[Any] 			=    list(_lowerCAmelCase			)
           __snake_case       :				Any 			=    conv_bias
           __snake_case       :				Tuple 			=    num_conv_pos_embeddings
           __snake_case       :				List[Any] 			=    num_conv_pos_embedding_groups
           __snake_case       :				Union[str, Any] 			=    len(self.conv_dim			)
           __snake_case       :				List[Any] 			=    num_hidden_layers
           __snake_case       :				Tuple 			=    intermediate_size
           __snake_case       :				Optional[int] 			=    hidden_act
           __snake_case       :				Union[str, Any] 			=    num_attention_heads
           __snake_case       :				str 			=    hidden_dropout
           __snake_case       :				Union[str, Any] 			=    attention_dropout
           __snake_case       :				Dict 			=    activation_dropout
           __snake_case       :				List[Any] 			=    feat_proj_layer_norm
           __snake_case       :				List[str] 			=    feat_proj_dropout
           __snake_case       :				List[Any] 			=    final_dropout
           __snake_case       :				str 			=    layerdrop
           __snake_case       :				List[str] 			=    layer_norm_eps
           __snake_case       :				Optional[Any] 			=    initializer_range
           __snake_case       :				Optional[int] 			=    vocab_size
           __snake_case       :				Dict 			=    do_stable_layer_norm
           __snake_case       :				Tuple 			=    use_weighted_layer_sum
           __snake_case       :				List[Any] 			=    classifier_proj_size

           if (
               (len(self.conv_stride			) != self.num_feat_extract_layers)
               or (len(self.conv_kernel			) != self.num_feat_extract_layers)
               or (len(self.conv_dim			) != self.num_feat_extract_layers)
           ):
               raise ValueError(
                   """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                   """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                   f''' {len(self.conv_dim			)}`, `len(config.conv_stride) = {len(self.conv_stride			)}`,'''
                   f''' `len(config.conv_kernel) = {len(self.conv_kernel			)}`.'''			)

           # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
           __snake_case       :				Union[str, Any] 			=    apply_spec_augment
           __snake_case       :				List[str] 			=    mask_time_prob
           __snake_case       :				Dict 			=    mask_time_length
           __snake_case       :				int 			=    mask_time_min_masks
           __snake_case       :				List[str] 			=    mask_feature_prob
           __snake_case       :				Tuple 			=    mask_feature_length
           __snake_case       :				int 			=    mask_feature_min_masks

           # ctc loss
           __snake_case       :				Dict 			=    ctc_loss_reduction
           __snake_case       :				List[str] 			=    ctc_zero_infinity





       @property
       def 			snake_case__    (							self					:     Any			):
           return functools.reduce(operator.mul			,			self.conv_stride			,			1			)



