



'''simple docstring'''


import torch
from torch import nn

class  SCREAMING_SNAKE_CASE (  nn.Module		):






    """simple docstring"""





    def __init__(  self      :					List[Any]  ,						UpperCamelCase__      :					Dict  ,						UpperCamelCase__      :					List[Any]  ,						UpperCamelCase__      :					List[str]  ,						UpperCamelCase__      :					Dict  ,						UpperCamelCase__      :					Any=1  ,						UpperCamelCase__      :					int=False ):


       """simple docstring"""







       super().__init__()

       UpperCamelCase					=  n_token
       UpperCamelCase					=  d_embed
       UpperCamelCase					=  d_proj

       UpperCamelCase					=  cutoffs + [n_token]
       UpperCamelCase					=  [0] + self.cutoffs
       UpperCamelCase					=  div_val

       UpperCamelCase					=  self.cutoffs[0]
       UpperCamelCase					=  len(self.cutoffs ) - 1
       UpperCamelCase					=  self.shortlist_size + self.n_clusters

       if self.n_clusters > 0:
          UpperCamelCase					=  nn.Parameter(torch.zeros(self.n_clusters  ,						self.d_embed ) )
          UpperCamelCase					=  nn.Parameter(torch.zeros(self.n_clusters ) )

       UpperCamelCase					=  nn.ModuleList()
       UpperCamelCase					=  nn.ParameterList()

       if div_val == 1:
          for i in range(len(self.cutoffs ) ):
             if d_proj != d_embed:
                self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase__  ,						UpperCamelCase__ ) ) )
             else:
                self.out_projs.append(UpperCamelCase__ )

          self.out_layers.append(nn.Linear(UpperCamelCase__  ,						UpperCamelCase__ ) )
       else:
          for i in range(len(self.cutoffs ) ):
             UpperCamelCase				,      UpperCamelCase					=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
             UpperCamelCase					=  d_embed // (div_val**i)

             self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCamelCase__  ,						UpperCamelCase__ ) ) )

             self.out_layers.append(nn.Linear(UpperCamelCase__  ,						r_idx - l_idx ) )

       UpperCamelCase					=  keep_order






    def  A       (  self      :					str  ,						UpperCamelCase__      :					List[str]  ,						UpperCamelCase__      :					Optional[int]  ,						UpperCamelCase__      :					int  ,						UpperCamelCase__      :					List[Any] ):


       """simple docstring"""







       if proj is None:
          UpperCamelCase					=  nn.functional.linear(UpperCamelCase__  ,						UpperCamelCase__  ,						bias=UpperCamelCase__ )
       else:
          # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
          UpperCamelCase					=  nn.functional.linear(UpperCamelCase__  ,						proj.t().contiguous() )
          UpperCamelCase					=  nn.functional.linear(UpperCamelCase__  ,						UpperCamelCase__  ,						bias=UpperCamelCase__ )
          # else:
          #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
          #     if bias is not None:
          #         logit = logit + bias

       return logit






    def  A       (  self      :					int  ,						UpperCamelCase__      :					Dict  ,						UpperCamelCase__      :					Optional[int]=None  ,						UpperCamelCase__      :					str=False ):


       """simple docstring"""







       if labels is not None:
          # Shift so that tokens < n predict n
          UpperCamelCase					=  hidden[..., :-1, :].contiguous()
          UpperCamelCase					=  labels[..., 1:].contiguous()
          UpperCamelCase					=  hidden.view(-1  ,						hidden.size(-1 ) )
          UpperCamelCase					=  labels.view(-1 )
          if hidden.size(0 ) != labels.size(0 ):
             raise RuntimeError('Input and labels should have the same size in the batch dimension.' )
       else:
          UpperCamelCase					=  hidden.view(-1  ,						hidden.size(-1 ) )

       if self.n_clusters == 0:
          UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						self.out_layers[0].weight  ,						self.out_layers[0].bias  ,						self.out_projs[0] )
          if labels is not None:
             UpperCamelCase					=  labels != -1_0_0
             UpperCamelCase					=  torch.zeros_like(UpperCamelCase__  ,						dtype=hidden.dtype  ,						device=hidden.device )
             UpperCamelCase					=  (
                 -nn.functional.log_softmax(UpperCamelCase__  ,						dim=-1 )[mask].gather(1  ,						labels[mask].unsqueeze(1 ) ).squeeze(1 )
             )
          else:
             UpperCamelCase					=  nn.functional.log_softmax(UpperCamelCase__  ,						dim=-1 )
       else:
          # construct weights and biases
          UpperCamelCase				,      UpperCamelCase					=  [], []
          for i in range(len(self.cutoffs ) ):
             if self.div_val == 1:
                UpperCamelCase				,      UpperCamelCase					=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                UpperCamelCase					=  self.out_layers[0].weight[l_idx:r_idx]
                UpperCamelCase					=  self.out_layers[0].bias[l_idx:r_idx]
             else:
                UpperCamelCase					=  self.out_layers[i].weight
                UpperCamelCase					=  self.out_layers[i].bias

             if i == 0:
                UpperCamelCase					=  torch.cat([weight_i, self.cluster_weight]  ,						dim=0 )
                UpperCamelCase					=  torch.cat([bias_i, self.cluster_bias]  ,						dim=0 )

             weights.append(UpperCamelCase__ )
             biases.append(UpperCamelCase__ )

          UpperCamelCase				,      UpperCamelCase				,      UpperCamelCase					=  weights[0], biases[0], self.out_projs[0]

          UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )
          UpperCamelCase					=  nn.functional.log_softmax(UpperCamelCase__  ,						dim=1 )

          if labels is None:
             UpperCamelCase					=  hidden.new_empty((head_logit.size(0 ), self.n_token) )
          else:
             UpperCamelCase					=  torch.zeros_like(UpperCamelCase__  ,						dtype=hidden.dtype  ,						device=hidden.device )

          UpperCamelCase					=  0
          UpperCamelCase					=  [0] + self.cutoffs
          for i in range(len(UpperCamelCase__ ) - 1 ):
             UpperCamelCase				,      UpperCamelCase					=  cutoff_values[i], cutoff_values[i + 1]

             if labels is not None:
                UpperCamelCase					=  (labels >= l_idx) & (labels < r_idx)
                UpperCamelCase					=  mask_i.nonzero().squeeze()

                if indices_i.numel() == 0:
                   continue

                UpperCamelCase					=  labels.index_select(0  ,						UpperCamelCase__ ) - l_idx
                UpperCamelCase					=  head_logprob.index_select(0  ,						UpperCamelCase__ )
                UpperCamelCase					=  hidden.index_select(0  ,						UpperCamelCase__ )
             else:
                UpperCamelCase					=  hidden

             if i == 0:
                if labels is not None:
                   UpperCamelCase					=  head_logprob_i.gather(1  ,						target_i[:, None] ).squeeze(1 )
                else:
                   UpperCamelCase					=  head_logprob[:, : self.cutoffs[0]]
             else:
                UpperCamelCase				,      UpperCamelCase				,      UpperCamelCase					=  weights[i], biases[i], self.out_projs[i]

                UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )
                UpperCamelCase					=  nn.functional.log_softmax(UpperCamelCase__  ,						dim=1 )
                UpperCamelCase					=  self.cutoffs[0] + i - 1  # No probability for the head cluster
                if labels is not None:
                   UpperCamelCase					=  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                       1  ,						target_i[:, None] ).squeeze(1 )
                else:
                   UpperCamelCase					=  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                   UpperCamelCase					=  logprob_i

             if labels is not None:
                if (hasattr(self  ,						'keep_order' ) and self.keep_order) or keep_order:
                   out.index_copy_(0  ,						UpperCamelCase__  ,						-logprob_i )
                else:
                   out[offset : offset + logprob_i.size(0 )].copy_(-logprob_i )
                offset += logprob_i.size(0 )

       return out






    def  A       (  self      :					List[Any]  ,						UpperCamelCase__      :					str ):


       """simple docstring"""







       if self.n_clusters == 0:
          UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						self.out_layers[0].weight  ,						self.out_layers[0].bias  ,						self.out_projs[0] )
          return nn.functional.log_softmax(UpperCamelCase__  ,						dim=-1 )
       else:
          # construct weights and biases
          UpperCamelCase				,      UpperCamelCase					=  [], []
          for i in range(len(self.cutoffs ) ):
             if self.div_val == 1:
                UpperCamelCase				,      UpperCamelCase					=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                UpperCamelCase					=  self.out_layers[0].weight[l_idx:r_idx]
                UpperCamelCase					=  self.out_layers[0].bias[l_idx:r_idx]
             else:
                UpperCamelCase					=  self.out_layers[i].weight
                UpperCamelCase					=  self.out_layers[i].bias

             if i == 0:
                UpperCamelCase					=  torch.cat([weight_i, self.cluster_weight]  ,						dim=0 )
                UpperCamelCase					=  torch.cat([bias_i, self.cluster_bias]  ,						dim=0 )

             weights.append(UpperCamelCase__ )
             biases.append(UpperCamelCase__ )

          UpperCamelCase				,      UpperCamelCase				,      UpperCamelCase					=  weights[0], biases[0], self.out_projs[0]
          UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )

          UpperCamelCase					=  hidden.new_empty((head_logit.size(0 ), self.n_token) )
          UpperCamelCase					=  nn.functional.log_softmax(UpperCamelCase__  ,						dim=1 )

          UpperCamelCase					=  [0] + self.cutoffs
          for i in range(len(UpperCamelCase__ ) - 1 ):
             UpperCamelCase				,      UpperCamelCase					=  cutoff_values[i], cutoff_values[i + 1]

             if i == 0:
                UpperCamelCase					=  head_logprob[:, : self.cutoffs[0]]
             else:
                UpperCamelCase				,      UpperCamelCase				,      UpperCamelCase					=  weights[i], biases[i], self.out_projs[i]

                UpperCamelCase					=  self._compute_logit(UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__  ,						UpperCamelCase__ )
                UpperCamelCase					=  nn.functional.log_softmax(UpperCamelCase__  ,						dim=1 )

                UpperCamelCase					=  head_logprob[:, -i] + tail_logprob_i
                UpperCamelCase					=  logprob_i

          return out






'''simple docstring'''


_lowerCamelCase	:       int      	=	"0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
   from .utils import rich

