



'''simple docstring'''





import torch
from torch import nn







class     UpperCAmelCase_       (       nn.Module	):
    """simple docstring"""







    def __init__(    self					:     Optional[int] ,     snake_case_					:     List[str] ,     snake_case_					:     Any ,     snake_case_					:     Union[str, Any] ,     snake_case_					:     Union[str, Any] ,     snake_case_					:     Optional[Any]=1 ,     snake_case_					:     Tuple=False						):
          super().__init__()

          snake_case__  :    List[str]    						=      n_token
          snake_case__  :    Optional[int]    						=      d_embed
          snake_case__  :    Any    						=      d_proj

          snake_case__  :    Any    						=      cutoffs + [n_token]
          snake_case__  :    Optional[int]    						=      [0] + self.cutoffs
          snake_case__  :    Union[str, Any]    						=      div_val

          snake_case__  :    Optional[Any]    						=      self.cutoffs[0]
          snake_case__  :    int    						=      len(self.cutoffs						) - 1
          snake_case__  :    int    						=      self.shortlist_size + self.n_clusters

          if self.n_clusters > 0:
                snake_case__  :    Optional[Any]    						=      nn.Parameter(torch.zeros(self.n_clusters ,     self.d_embed						)						)
                snake_case__  :    List[str]    						=      nn.Parameter(torch.zeros(self.n_clusters						)						)

          snake_case__  :    Any    						=      nn.ModuleList()
          snake_case__  :    List[str]    						=      nn.ParameterList()

          if div_val == 1:
                for i in range(len(self.cutoffs						)						):
                      if d_proj != d_embed:
                            self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case_ ,     snake_case_						)						)						)
                      else:
                            self.out_projs.append(snake_case_						)

                self.out_layers.append(nn.Linear(snake_case_ ,     snake_case_						)						)
          else:
                for i in range(len(self.cutoffs						)						):
                      snake_case__						,				snake_case__  :    int    						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                      snake_case__  :    List[Any]    						=      d_embed // (div_val**i)

                      self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case_ ,     snake_case_						)						)						)

                      self.out_layers.append(nn.Linear(snake_case_ ,     r_idx - l_idx						)						)

          snake_case__  :    int    						=      keep_order




    def lowerCamelCase       (    self					:     Dict ,     snake_case_					:     Union[str, Any] ,     snake_case_					:     List[Any] ,     snake_case_					:     Tuple ,     snake_case_					:     List[str]						):
          if proj is None:
                snake_case__  :    List[str]    						=      nn.functional.linear(snake_case_ ,     snake_case_ ,     bias=snake_case_						)
          else:
                # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
                snake_case__  :    int    						=      nn.functional.linear(snake_case_ ,     proj.t().contiguous()						)
                snake_case__  :    Optional[Any]    						=      nn.functional.linear(snake_case_ ,     snake_case_ ,     bias=snake_case_						)
                # else:
                #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
                #     if bias is not None:
                #         logit = logit + bias

          return logit




    def lowerCamelCase       (    self					:     Tuple ,     snake_case_					:     Union[str, Any] ,     snake_case_					:     Tuple=None ,     snake_case_					:     str=False						):

          if labels is not None:
                # Shift so that tokens < n predict n
                snake_case__  :    Dict    						=      hidden[..., :-1, :].contiguous()
                snake_case__  :    Any    						=      labels[..., 1:].contiguous()
                snake_case__  :    Tuple    						=      hidden.view(-1 ,     hidden.size(-1						)						)
                snake_case__  :    Optional[Any]    						=      labels.view(-1						)
                if hidden.size(0						) != labels.size(0						):
                      raise RuntimeError("""Input and labels should have the same size in the batch dimension."""						)
          else:
                snake_case__  :    List[Any]    						=      hidden.view(-1 ,     hidden.size(-1						)						)

          if self.n_clusters == 0:
                snake_case__  :    str    						=      self._compute_logit(snake_case_ ,     self.out_layers[0].weight ,     self.out_layers[0].bias ,     self.out_projs[0]						)
                if labels is not None:
                      snake_case__  :    Dict    						=      labels != -100
                      snake_case__  :    Any    						=      torch.zeros_like(snake_case_ ,     dtype=hidden.dtype ,     device=hidden.device						)
                      snake_case__  :    int    						=      (
                          -nn.functional.log_softmax(snake_case_ ,     dim=-1						)[mask].gather(1 ,     labels[mask].unsqueeze(1						)						).squeeze(1						)
                      )
                else:
                      snake_case__  :    str    						=      nn.functional.log_softmax(snake_case_ ,     dim=-1						)
          else:
                # construct weights and biases
                snake_case__						,				snake_case__  :    Optional[Any]    						=      [], []
                for i in range(len(self.cutoffs						)						):
                      if self.div_val == 1:
                            snake_case__						,				snake_case__  :    str    						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                            snake_case__  :    Optional[Any]    						=      self.out_layers[0].weight[l_idx:r_idx]
                            snake_case__  :    int    						=      self.out_layers[0].bias[l_idx:r_idx]
                      else:
                            snake_case__  :    Any    						=      self.out_layers[i].weight
                            snake_case__  :    Optional[int]    						=      self.out_layers[i].bias

                      if i == 0:
                            snake_case__  :    List[Any]    						=      torch.cat([weight_i, self.cluster_weight] ,     dim=0						)
                            snake_case__  :    Union[str, Any]    						=      torch.cat([bias_i, self.cluster_bias] ,     dim=0						)

                      weights.append(snake_case_						)
                      biases.append(snake_case_						)

                snake_case__						,				snake_case__						,				snake_case__  :    Any    						=      weights[0], biases[0], self.out_projs[0]

                snake_case__  :    Dict    						=      self._compute_logit(snake_case_ ,     snake_case_ ,     snake_case_ ,     snake_case_						)
                snake_case__  :    List[str]    						=      nn.functional.log_softmax(snake_case_ ,     dim=1						)

                if labels is None:
                      snake_case__  :    Union[str, Any]    						=      hidden.new_empty((head_logit.size(0						), self.n_token)						)
                else:
                      snake_case__  :    Optional[int]    						=      torch.zeros_like(snake_case_ ,     dtype=hidden.dtype ,     device=hidden.device						)

                snake_case__  :    List[str]    						=      0
                snake_case__  :    Dict    						=      [0] + self.cutoffs
                for i in range(len(snake_case_						) - 1						):
                      snake_case__						,				snake_case__  :    int    						=      cutoff_values[i], cutoff_values[i + 1]

                      if labels is not None:
                            snake_case__  :    str    						=      (labels >= l_idx) & (labels < r_idx)
                            snake_case__  :    Union[str, Any]    						=      mask_i.nonzero().squeeze()

                            if indices_i.numel() == 0:
                                  continue

                            snake_case__  :    List[Any]    						=      labels.index_select(0 ,     snake_case_						) - l_idx
                            snake_case__  :    Dict    						=      head_logprob.index_select(0 ,     snake_case_						)
                            snake_case__  :    List[str]    						=      hidden.index_select(0 ,     snake_case_						)
                      else:
                            snake_case__  :    List[str]    						=      hidden

                      if i == 0:
                            if labels is not None:
                                  snake_case__  :    str    						=      head_logprob_i.gather(1 ,     target_i[:, None]						).squeeze(1						)
                            else:
                                  snake_case__  :    Tuple    						=      head_logprob[:, : self.cutoffs[0]]
                      else:
                            snake_case__						,				snake_case__						,				snake_case__  :    Dict    						=      weights[i], biases[i], self.out_projs[i]

                            snake_case__  :    Optional[Any]    						=      self._compute_logit(snake_case_ ,     snake_case_ ,     snake_case_ ,     snake_case_						)
                            snake_case__  :    Union[str, Any]    						=      nn.functional.log_softmax(snake_case_ ,     dim=1						)
                            snake_case__  :    List[Any]    						=      self.cutoffs[0] + i - 1  # No probability for the head cluster
                            if labels is not None:
                                  snake_case__  :    str    						=      head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                      1 ,     target_i[:, None]						).squeeze(1						)
                            else:
                                  snake_case__  :    List[str]    						=      head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                  snake_case__  :    List[Any]    						=      logprob_i

                      if labels is not None:
                            if (hasattr(self ,     """keep_order"""						) and self.keep_order) or keep_order:
                                  out.index_copy_(0 ,     snake_case_ ,     -logprob_i						)
                            else:
                                  out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
                            offset += logprob_i.size(0						)

          return out




    def lowerCamelCase       (    self					:     Union[str, Any] ,     snake_case_					:     Dict						):
          if self.n_clusters == 0:
                snake_case__  :    int    						=      self._compute_logit(snake_case_ ,     self.out_layers[0].weight ,     self.out_layers[0].bias ,     self.out_projs[0]						)
                return nn.functional.log_softmax(snake_case_ ,     dim=-1						)
          else:
                # construct weights and biases
                snake_case__						,				snake_case__  :    Union[str, Any]    						=      [], []
                for i in range(len(self.cutoffs						)						):
                      if self.div_val == 1:
                            snake_case__						,				snake_case__  :    Dict    						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                            snake_case__  :    Union[str, Any]    						=      self.out_layers[0].weight[l_idx:r_idx]
                            snake_case__  :    int    						=      self.out_layers[0].bias[l_idx:r_idx]
                      else:
                            snake_case__  :    List[Any]    						=      self.out_layers[i].weight
                            snake_case__  :    List[str]    						=      self.out_layers[i].bias

                      if i == 0:
                            snake_case__  :    Union[str, Any]    						=      torch.cat([weight_i, self.cluster_weight] ,     dim=0						)
                            snake_case__  :    Tuple    						=      torch.cat([bias_i, self.cluster_bias] ,     dim=0						)

                      weights.append(snake_case_						)
                      biases.append(snake_case_						)

                snake_case__						,				snake_case__						,				snake_case__  :    Union[str, Any]    						=      weights[0], biases[0], self.out_projs[0]
                snake_case__  :    List[str]    						=      self._compute_logit(snake_case_ ,     snake_case_ ,     snake_case_ ,     snake_case_						)

                snake_case__  :    int    						=      hidden.new_empty((head_logit.size(0						), self.n_token)						)
                snake_case__  :    Optional[Any]    						=      nn.functional.log_softmax(snake_case_ ,     dim=1						)

                snake_case__  :    Tuple    						=      [0] + self.cutoffs
                for i in range(len(snake_case_						) - 1						):
                      snake_case__						,				snake_case__  :    Optional[int]    						=      cutoff_values[i], cutoff_values[i + 1]

                      if i == 0:
                            snake_case__  :    int    						=      head_logprob[:, : self.cutoffs[0]]
                      else:
                            snake_case__						,				snake_case__						,				snake_case__  :    Optional[int]    						=      weights[i], biases[i], self.out_projs[i]

                            snake_case__  :    str    						=      self._compute_logit(snake_case_ ,     snake_case_ ,     snake_case_ ,     snake_case_						)
                            snake_case__  :    Optional[int]    						=      nn.functional.log_softmax(snake_case_ ,     dim=1						)

                            snake_case__  :    Union[str, Any]    						=      head_logprob[:, -i] + tail_logprob_i
                            snake_case__  :    int    						=      logprob_i

                return out


"""simple docstring"""




import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   ArgumentParser(
							    description=(
							        """PyTorch TPU distributed training launch """
							        """helper utility that will spawn up """
							        """multiple distributed processes"""
							    )						)

							# Optional arguments for the launch helper
							parser.add_argument("""--num_cores"""   ,type=_snake_case   ,default=1   ,help="""Number of TPU cores to use (1 or 8)."""						)

							# positional
							parser.add_argument(
							    """training_script"""   ,type=_snake_case   ,help=(
							        """The full path to the single TPU training """
							        """program/script to be launched in parallel, """
							        """followed by all the arguments for the """
							        """training script"""
							    )   ,)

							# rest from the training program
							parser.add_argument("""training_script_args"""   ,nargs=_snake_case						)

							return parser.parse_args()

def lowercase_  (   ):
							SCREAMING_SNAKE_CASE__						:       int           =   parse_args()

							# Import training_script as a module.
							SCREAMING_SNAKE_CASE__						:       Dict           =   Path(args.training_script						)
							sys.path.append(str(script_fpath.parent.resolve()						)						)
							SCREAMING_SNAKE_CASE__						:       int           =   script_fpath.stem
							SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   importlib.import_module(_snake_case						)

							# Patch sys.argv
							SCREAMING_SNAKE_CASE__						:       str           =   [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores						)]
							xmp.spawn(mod._mp_fn   ,args=()   ,nprocs=args.num_cores						)


if __name__ == "__main__":
					main()




