




from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class 		UpperCamelCase__	(__SCREAMING_SNAKE_CASE    ,				__SCREAMING_SNAKE_CASE    ,				__SCREAMING_SNAKE_CASE     ):

    '''simple docstring'''



    lowerCamelCase_      :	Union[str, Any]	     =      [R"""h\.\d+\.attn\.bias""", R"""h\.\d+\.attn\.masked_bias"""]







    @register_to_config
    def __init__(    self     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = 5_0257     ,  UpperCamelCase__ = 1024     ,  UpperCamelCase__ = 768     ,  UpperCamelCase__ = 12     ,  UpperCamelCase__ = 12     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = "gelu_new"     ,  UpperCamelCase__ = 0.1     ,  UpperCamelCase__ = 0.1     ,  UpperCamelCase__ = 0.1     ,  UpperCamelCase__ = 1e-5     ,  UpperCamelCase__ = 0.02     ,  UpperCamelCase__ = True     ,  UpperCamelCase__ = True     ,  UpperCamelCase__ = False     ,  UpperCamelCase__ = False     ,  )     ->      str:
           super().__init__()

           lowerCamelCase   :      List[str]			 =					prefix_length

           if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
                  raise ValueError(
                      F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
                      F''' `n_embd`: {n_embd} are not equal.'''     )

           lowerCamelCase   :      List[str]			 =					prefix_inner_dim
           lowerCamelCase   :      Union[str, Any]			 =					prefix_hidden_dim

           lowerCamelCase   :      Tuple			 =					(
               nn.Linear(self.prefix_inner_dim     ,  self.prefix_hidden_dim     )
               if self.prefix_hidden_dim is not None
               else nn.Identity()
           )
           lowerCamelCase   :      Any			 =					(
               nn.Linear(self.prefix_hidden_dim     ,  UpperCAmelCase_     ) if self.prefix_hidden_dim is not None else nn.Identity()
           )

           lowerCamelCase   :      Dict			 =					GPTaConfig(
               vocab_size=UpperCAmelCase_     ,  n_positions=UpperCAmelCase_     ,  n_embd=UpperCAmelCase_     ,  n_layer=UpperCAmelCase_     ,  n_head=UpperCAmelCase_     ,  n_inner=UpperCAmelCase_     ,  activation_function=UpperCAmelCase_     ,  resid_pdrop=UpperCAmelCase_     ,  embd_pdrop=UpperCAmelCase_     ,  attn_pdrop=UpperCAmelCase_     ,  layer_norm_epsilon=UpperCAmelCase_     ,  initializer_range=UpperCAmelCase_     ,  scale_attn_weights=UpperCAmelCase_     ,  use_cache=UpperCAmelCase_     ,  scale_attn_by_inverse_layer_idx=UpperCAmelCase_     ,  reorder_and_upcast_attn=UpperCAmelCase_     ,  )
           lowerCamelCase   :      int			 =					GPTaLMHeadModel(UpperCAmelCase_     )







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     ,  UpperCamelCase__ = None     ,  )     ->      Optional[Any]:
           lowerCamelCase   :      str			 =					self.transformer.transformer.wte(UpperCAmelCase_     )
           lowerCamelCase   :      List[str]			 =					self.encode_prefix(UpperCAmelCase_     )
           lowerCamelCase   :      Optional[int]			 =					self.decode_prefix(UpperCAmelCase_     )
           lowerCamelCase   :      List[str]			 =					torch.cat((prefix_embeds, embedding_text)     ,  dim=1     )

           if labels is not None:
                  lowerCamelCase   :      List[Any]			 =					self.get_dummy_token(input_ids.shape[0]     ,  input_ids.device     )
                  lowerCamelCase   :      List[Any]			 =					torch.cat((dummy_token, input_ids)     ,  dim=1     )
           lowerCamelCase   :      Tuple			 =					self.transformer(inputs_embeds=UpperCAmelCase_     ,  labels=UpperCAmelCase_     ,  attention_mask=UpperCAmelCase_     )
           if self.prefix_hidden_dim is not None:
                  return out, hidden
           else:
                  return out







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__     )     ->      torch.Tensor:
           return torch.zeros(UpperCAmelCase_     ,  self.prefix_length     ,  dtype=torch.intaa     ,  device=UpperCAmelCase_     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Any:
           return self.encode_prefix(UpperCAmelCase_     )







    @torch.no_grad()
    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     )     ->      Optional[int]:
           lowerCamelCase   :      Optional[Any]			 =					torch.split(UpperCAmelCase_     ,  1     ,  dim=0     )
           lowerCamelCase   :      Dict			 =					[]
           lowerCamelCase   :      List[Any]			 =					[]
           for feature in features:
                  lowerCamelCase   :      Tuple			 =					self.decode_prefix(feature.to(UpperCAmelCase_     )     )  # back to the clip feature
                  # Only support beam search for now
                  lowerCamelCase   :      Tuple			 =					self.generate_beam(
                      input_embeds=UpperCAmelCase_     ,  device=UpperCAmelCase_     ,  eos_token_id=UpperCAmelCase_     )
                  generated_tokens.append(output_tokens[0]     )
                  generated_seq_lengths.append(seq_lengths[0]     )
           lowerCamelCase   :      Union[str, Any]			 =					torch.stack(UpperCAmelCase_     )
           lowerCamelCase   :      List[Any]			 =					torch.stack(UpperCAmelCase_     )
           return generated_tokens, generated_seq_lengths







    @torch.no_grad()
    def     _lowercase     (    self     ,  UpperCamelCase__=None     ,  UpperCamelCase__=None     ,  UpperCamelCase__=None     ,  UpperCamelCase__ = 5     ,  UpperCamelCase__ = 67     ,  UpperCamelCase__ = 1.0     ,  UpperCamelCase__ = None     ,  )     ->      List[Any]:
           lowerCamelCase   :      List[str]			 =					eos_token_id
           lowerCamelCase   :      str			 =					None
           lowerCamelCase   :      Optional[Any]			 =					None
           lowerCamelCase   :      Any			 =					torch.ones(UpperCAmelCase_     ,  device=UpperCAmelCase_     ,  dtype=torch.int     )
           lowerCamelCase   :      Any			 =					torch.zeros(UpperCAmelCase_     ,  device=UpperCAmelCase_     ,  dtype=torch.bool     )

           if input_embeds is not None:
                  lowerCamelCase   :      Union[str, Any]			 =					input_embeds
           else:
                  lowerCamelCase   :      Optional[int]			 =					self.transformer.transformer.wte(UpperCAmelCase_     )

           for i in range(UpperCAmelCase_     ):
                  lowerCamelCase   :      Dict			 =					self.transformer(inputs_embeds=UpperCAmelCase_     )
                  lowerCamelCase   :      Optional[Any]			 =					outputs.logits
                  lowerCamelCase   :      Dict			 =					logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
                  lowerCamelCase   :      Dict			 =					logits.softmax(-1     ).log()

                  if scores is None:
                         lowerCamelCase   :      str			 =					logits.topk(UpperCAmelCase_     ,  -1     )
                         lowerCamelCase   :      List[str]			 =					generated.expand(UpperCAmelCase_     ,  *generated.shape[1:]     )
                         lowerCamelCase   :      List[str]			 =					next_tokens.permute(1     ,  0     ), scores.squeeze(0     )
                         if tokens is None:
                                lowerCamelCase   :      Optional[int]			 =					next_tokens
                         else:
                                lowerCamelCase   :      Optional[Any]			 =					tokens.expand(UpperCAmelCase_     ,  *tokens.shape[1:]     )
                                lowerCamelCase   :      List[str]			 =					torch.cat((tokens, next_tokens)     ,  dim=1     )
                  else:
                         lowerCamelCase   :      Any			 =					-float(np.inf     )
                         lowerCamelCase   :      Optional[Any]			 =					0
                         lowerCamelCase   :      Optional[int]			 =					scores[:, None] + logits
                         seq_lengths[~is_stopped] += 1
                         lowerCamelCase   :      int			 =					scores_sum / seq_lengths[:, None]
                         lowerCamelCase   :      Any			 =					scores_sum_average.view(-1     ).topk(UpperCAmelCase_     ,  -1     )
                         lowerCamelCase   :      List[Any]			 =					next_tokens // scores_sum.shape[1]
                         lowerCamelCase   :      str			 =					seq_lengths[next_tokens_source]
                         lowerCamelCase   :      Optional[int]			 =					next_tokens % scores_sum.shape[1]
                         lowerCamelCase   :      Optional[Any]			 =					next_tokens.unsqueeze(1     )
                         lowerCamelCase   :      Dict			 =					tokens[next_tokens_source]
                         lowerCamelCase   :      Union[str, Any]			 =					torch.cat((tokens, next_tokens)     ,  dim=1     )
                         lowerCamelCase   :      List[Any]			 =					generated[next_tokens_source]
                         lowerCamelCase   :      List[str]			 =					scores_sum_average * seq_lengths
                         lowerCamelCase   :      Tuple			 =					is_stopped[next_tokens_source]

                  lowerCamelCase   :      Dict			 =					self.transformer.transformer.wte(next_tokens.squeeze()     ).view(generated.shape[0]     ,  1     ,  -1     )
                  lowerCamelCase   :      List[str]			 =					torch.cat((generated, next_token_embed)     ,  dim=1     )
                  lowerCamelCase   :      List[Any]			 =					is_stopped + next_tokens.eq(UpperCAmelCase_     ).squeeze()
                  if is_stopped.all():
                         break

           lowerCamelCase   :      Dict			 =					scores / seq_lengths
           lowerCamelCase   :      Dict			 =					scores.argsort(descending=UpperCAmelCase_     )
           # tokens tensors are already padded to max_seq_length
           lowerCamelCase   :      Tuple			 =					[tokens[i] for i in order]
           lowerCamelCase   :      List[Any]			 =					torch.stack(UpperCAmelCase_     ,  dim=0     )
           lowerCamelCase   :      Optional[Any]			 =					torch.tensor([seq_lengths[i] for i in order]     ,  dtype=seq_lengths.dtype     )
           return output_texts, seq_lengths













import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A									=				logging.get_logger(__name__)

__A									=				{
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







 '''simple docstring'''
 lowercase_		=			"unispeech"

 def __init__(self				:   Any       , UpperCAmelCase_				:   Any=32       , UpperCAmelCase_				:   List[str]=768       , UpperCAmelCase_				:   Any=12       , UpperCAmelCase_				:   Union[str, Any]=12       , UpperCAmelCase_				:   Optional[Any]=3_072       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   int=0.1       , UpperCAmelCase_				:   Any=0.0       , UpperCAmelCase_				:   str=0.0       , UpperCAmelCase_				:   Dict=0.1       , UpperCAmelCase_				:   Optional[int]=0.1       , UpperCAmelCase_				:   Optional[Any]=0.02       , UpperCAmelCase_				:   Union[str, Any]=1E-5       , UpperCAmelCase_				:   str="group"       , UpperCAmelCase_				:   List[Any]="gelu"       , UpperCAmelCase_				:   Tuple=(512, 512, 512, 512, 512, 512, 512)       , UpperCAmelCase_				:   str=(5, 2, 2, 2, 2, 2, 2)       , UpperCAmelCase_				:   Any=(10, 3, 3, 3, 3, 2, 2)       , UpperCAmelCase_				:   Optional[Any]=False       , UpperCAmelCase_				:   str=128       , UpperCAmelCase_				:   int=16       , UpperCAmelCase_				:   Dict=False       , UpperCAmelCase_				:   Optional[int]=True       , UpperCAmelCase_				:   Dict=0.05       , UpperCAmelCase_				:   Optional[int]=10       , UpperCAmelCase_				:   Tuple=2       , UpperCAmelCase_				:   Union[str, Any]=0.0       , UpperCAmelCase_				:   int=10       , UpperCAmelCase_				:   List[Any]=0       , UpperCAmelCase_				:   Optional[Any]=320       , UpperCAmelCase_				:   int=2       , UpperCAmelCase_				:   Union[str, Any]=0.1       , UpperCAmelCase_				:   str=100       , UpperCAmelCase_				:   Any=256       , UpperCAmelCase_				:   int=256       , UpperCAmelCase_				:   Optional[Any]=0.1       , UpperCAmelCase_				:   str="mean"       , UpperCAmelCase_				:   Union[str, Any]=False       , UpperCAmelCase_				:   List[str]=False       , UpperCAmelCase_				:   List[Any]=256       , UpperCAmelCase_				:   Optional[int]=80       , UpperCAmelCase_				:   Optional[int]=0       , UpperCAmelCase_				:   Optional[Any]=1       , UpperCAmelCase_				:   Union[str, Any]=2       , UpperCAmelCase_				:   Dict=0.5       , **UpperCAmelCase_				:   Optional[int]       , )       ->str:



   '''simple docstring'''



   super().__init__(**UpperCAmelCase_       , pad_token_id=UpperCAmelCase_       , bos_token_id=UpperCAmelCase_       , eos_token_id=UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =hidden_size
   lowerCamelCase__:      List[str]			  =feat_extract_norm
   lowerCamelCase__:      Dict			  =feat_extract_activation
   lowerCamelCase__:      Optional[Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Any			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Union[str, Any]			  =list(UpperCAmelCase_)
   lowerCamelCase__:      Dict			  =conv_bias
   lowerCamelCase__:      Optional[Any]			  =num_conv_pos_embeddings
   lowerCamelCase__:      Dict			  =num_conv_pos_embedding_groups
   lowerCamelCase__:      int			  =len(self.conv_dim)
   lowerCamelCase__:      Union[str, Any]			  =num_hidden_layers
   lowerCamelCase__:      Union[str, Any]			  =intermediate_size
   lowerCamelCase__:      Dict			  =hidden_act
   lowerCamelCase__:      List[Any]			  =num_attention_heads
   lowerCamelCase__:      Dict			  =hidden_dropout
   lowerCamelCase__:      Optional[Any]			  =attention_dropout
   lowerCamelCase__:      Optional[Any]			  =activation_dropout
   lowerCamelCase__:      Tuple			  =feat_proj_dropout
   lowerCamelCase__:      int			  =final_dropout
   lowerCamelCase__:      Optional[Any]			  =layerdrop
   lowerCamelCase__:      Dict			  =layer_norm_eps
   lowerCamelCase__:      Optional[Any]			  =initializer_range
   lowerCamelCase__:      int			  =num_ctc_classes
   lowerCamelCase__:      Tuple			  =vocab_size
   lowerCamelCase__:      Dict			  =do_stable_layer_norm
   lowerCamelCase__:      List[Any]			  =use_weighted_layer_sum
   lowerCamelCase__:      Dict			  =classifier_proj_size

   if (
       (len(self.conv_stride) != self.num_feat_extract_layers)
       or (len(self.conv_kernel) != self.num_feat_extract_layers)
       or (len(self.conv_dim) != self.num_feat_extract_layers)
   ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         F""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
         F""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   lowerCamelCase__:      int			  =apply_spec_augment
   lowerCamelCase__:      List[str]			  =mask_time_prob
   lowerCamelCase__:      Union[str, Any]			  =mask_time_length
   lowerCamelCase__:      List[Any]			  =mask_time_min_masks
   lowerCamelCase__:      Any			  =mask_feature_prob
   lowerCamelCase__:      Optional[Any]			  =mask_feature_length
   lowerCamelCase__:      List[str]			  =mask_feature_min_masks

   # parameters for pretraining with codevector quantized representations
   lowerCamelCase__:      Optional[Any]			  =num_codevectors_per_group
   lowerCamelCase__:      str			  =num_codevector_groups
   lowerCamelCase__:      Tuple			  =contrastive_logits_temperature
   lowerCamelCase__:      int			  =feat_quantizer_dropout
   lowerCamelCase__:      Any			  =num_negatives
   lowerCamelCase__:      List[str]			  =codevector_dim
   lowerCamelCase__:      Union[str, Any]			  =proj_codevector_dim
   lowerCamelCase__:      Any			  =diversity_loss_weight

   # ctc loss
   lowerCamelCase__:      Any			  =ctc_loss_reduction
   lowerCamelCase__:      Dict			  =ctc_zero_infinity

   # pretraining loss
   lowerCamelCase__:      Dict			  =replace_prob







 @property
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   return functools.reduce(operator.mul       , self.conv_stride       , 1)

