import tensorflow as tf

from ...tf_utils import shape_list



class   snake_case_       (		tf.keras.layers.Layer       ):






   def __init__(      self				:     Dict	,						lowercase_				:     Optional[Any]	,						lowercase_				:     Union[str, Any]	,						lowercase_				:     List[Any]	,						lowercase_				:     Dict	,						lowercase_				:     Dict=1	,						lowercase_				:     Any=False	,						**lowercase_				:     Dict  )					->						Optional[int]:
       super().__init__(**lowerCAmelCase_  )

       lowercase__ :   Optional[Any]    					=				vocab_size
       lowercase__ :   int    					=				d_embed
       lowercase__ :   Union[str, Any]    					=				d_proj

       lowercase__ :   Optional[Any]    					=				cutoffs + [vocab_size]
       lowercase__ :   Optional[Any]    					=				[0] + self.cutoffs
       lowercase__ :   int    					=				div_val

       lowercase__ :   str    					=				self.cutoffs[0]
       lowercase__ :   List[Any]    					=				len(self.cutoffs  ) - 1
       lowercase__ :   List[Any]    					=				self.shortlist_size + self.n_clusters
       lowercase__ :   Any    					=				keep_order

       lowercase__ :   Dict    					=				[]
       lowercase__ :   int    					=				[]






   def       __UpperCamelCase     (      self				:     Tuple	,						lowercase_				:     int  )					->						Union[str, Any]:






       if self.n_clusters > 0:
           lowercase__ :   Any    					=				self.add_weight(
               shape=(self.n_clusters, self.d_embed)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name="cluster_weight"  )
           lowercase__ :   Any    					=				self.add_weight(
               shape=(self.n_clusters,)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name="cluster_bias"  )

       if self.div_val == 1:
           for i in range(len(self.cutoffs  )  ):
               if self.d_proj != self.d_embed:
                   lowercase__ :   Optional[int]    					=				self.add_weight(
                       shape=(self.d_embed, self.d_proj)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_projs_._{i}'''	,						)
                   self.out_projs.append(lowerCAmelCase_  )
               else:
                   self.out_projs.append(lowerCAmelCase_  )
               lowercase__ :   Optional[int]    					=				self.add_weight(
                   shape=(self.vocab_size, self.d_embed)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_layers_._{i}_._weight'''	,						)
               lowercase__ :   List[str]    					=				self.add_weight(
                   shape=(self.vocab_size,)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_layers_._{i}_._bias'''	,						)
               self.out_layers.append((weight, bias)  )
       else:
           for i in range(len(self.cutoffs  )  ):
               lowercase__					,   lowercase__ :   Optional[Any]    					=				self.cutoff_ends[i], self.cutoff_ends[i + 1]
               lowercase__ :   List[Any]    					=				self.d_embed // (self.div_val**i)

               lowercase__ :   Any    					=				self.add_weight(
                   shape=(d_emb_i, self.d_proj)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_projs_._{i}'''  )
               self.out_projs.append(lowerCAmelCase_  )
               lowercase__ :   Tuple    					=				self.add_weight(
                   shape=(r_idx - l_idx, d_emb_i)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_layers_._{i}_._weight'''	,						)
               lowercase__ :   List[str]    					=				self.add_weight(
                   shape=(r_idx - l_idx,)	,						initializer="zeros"	,						trainable=lowerCAmelCase_	,						name=F'''out_layers_._{i}_._bias'''	,						)
               self.out_layers.append((weight, bias)  )
       super().build(lowerCAmelCase_  )






   @staticmethod
   def       __UpperCamelCase     (      lowercase_				:     int	,						lowercase_				:     Optional[int]	,						lowercase_				:     Optional[int]	,						lowercase_				:     Tuple=None  )					->						Any:
       lowercase__ :   Any    					=				x
       if proj is not None:
           lowercase__ :   str    					=				tf.einsum("ibd,ed->ibe"	,						lowerCAmelCase_	,						lowerCAmelCase_  )
       return tf.einsum("ibd,nd->ibn"	,						lowerCAmelCase_	,						lowerCAmelCase_  ) + b






   @staticmethod
   def       __UpperCamelCase     (      lowercase_				:     List[str]	,						lowercase_				:     Any  )					->						Optional[int]:
       lowercase__ :   Tuple    					=				shape_list(lowerCAmelCase_  )
       lowercase__ :   str    					=				tf.range(lp_size[0]	,						dtype=target.dtype  )
       lowercase__ :   str    					=				tf.stack([r, target]	,						1  )
       return tf.gather_nd(lowerCAmelCase_	,						lowerCAmelCase_  )







   def       __UpperCamelCase     (      self				:     Any	,						lowercase_				:     Union[str, Any]	,						lowercase_				:     int	,						lowercase_				:     List[str]=True	,						lowercase_				:     Union[str, Any]=False  )					->						Union[str, Any]:
       lowercase__ :   List[str]    					=				0
       if self.n_clusters == 0:
           lowercase__ :   Any    					=				self._logit(lowerCAmelCase_	,						self.out_layers[0][0]	,						self.out_layers[0][1]	,						self.out_projs[0]  )
           if target is not None:
               lowercase__ :   int    					=				tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCAmelCase_	,						logits=lowerCAmelCase_  )
           lowercase__ :   Optional[Any]    					=				tf.nn.log_softmax(lowerCAmelCase_	,						axis=-1  )
       else:
           lowercase__ :   Union[str, Any]    					=				shape_list(lowerCAmelCase_  )
           lowercase__ :   Dict    					=				[]
           lowercase__ :   Tuple    					=				tf.zeros(hidden_sizes[:2]  )
           for i in range(len(self.cutoffs  )  ):
               lowercase__					,   lowercase__ :   Union[str, Any]    					=				self.cutoff_ends[i], self.cutoff_ends[i + 1]
               if target is not None:
                   lowercase__ :   Optional[int]    					=				(target >= l_idx) & (target < r_idx)
                   lowercase__ :   Tuple    					=				tf.where(lowerCAmelCase_  )
                   lowercase__ :   Union[str, Any]    					=				tf.boolean_mask(lowerCAmelCase_	,						lowerCAmelCase_  ) - l_idx

               if self.div_val == 1:
                   lowercase__ :   Optional[Any]    					=				self.out_layers[0][0][l_idx:r_idx]
                   lowercase__ :   List[str]    					=				self.out_layers[0][1][l_idx:r_idx]
               else:
                   lowercase__ :   Any    					=				self.out_layers[i][0]
                   lowercase__ :   int    					=				self.out_layers[i][1]

               if i == 0:
                   lowercase__ :   Union[str, Any]    					=				tf.concat([cur_W, self.cluster_weight]	,						0  )
                   lowercase__ :   Optional[Any]    					=				tf.concat([cur_b, self.cluster_bias]	,						0  )

                   lowercase__ :   Any    					=				self._logit(lowerCAmelCase_	,						lowerCAmelCase_	,						lowerCAmelCase_	,						self.out_projs[0]  )
                   lowercase__ :   Optional[Any]    					=				tf.nn.log_softmax(lowerCAmelCase_  )
                   out.append(head_logprob[..., : self.cutoffs[0]]  )
                   if target is not None:
                       lowercase__ :   List[str]    					=				tf.boolean_mask(lowerCAmelCase_	,						lowerCAmelCase_  )
                       lowercase__ :   Any    					=				self._gather_logprob(lowerCAmelCase_	,						lowerCAmelCase_  )
               else:
                   lowercase__ :   List[Any]    					=				self._logit(lowerCAmelCase_	,						lowerCAmelCase_	,						lowerCAmelCase_	,						self.out_projs[i]  )
                   lowercase__ :   str    					=				tf.nn.log_softmax(lowerCAmelCase_  )
                   lowercase__ :   Dict    					=				self.cutoffs[0] + i - 1  # No probability for the head cluster
                   lowercase__ :   int    					=				head_logprob[..., cluster_prob_idx, None] + tail_logprob
                   out.append(lowerCAmelCase_  )
                   if target is not None:
                       lowercase__ :   Tuple    					=				tf.boolean_mask(lowerCAmelCase_	,						lowerCAmelCase_  )
                       lowercase__ :   Any    					=				tf.boolean_mask(lowerCAmelCase_	,						lowerCAmelCase_  )
                       lowercase__ :   Dict    					=				self._gather_logprob(lowerCAmelCase_	,						lowerCAmelCase_  )
                       cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
               if target is not None:
                   loss += tf.scatter_nd(lowerCAmelCase_	,						-cur_logprob	,						shape_list(lowerCAmelCase_  )  )
           lowercase__ :   Optional[Any]    					=				tf.concat(lowerCAmelCase_	,						axis=-1  )

       if target is not None:
           if return_mean:
               lowercase__ :   Optional[int]    					=				tf.reduce_mean(lowerCAmelCase_  )
           # Add the training-time loss value to the layer using `self.add_loss()`.
           self.add_loss(lowerCAmelCase_  )

           # Log the loss as a metric (we could log arbitrary metrics,
           # including different metrics for training and inference.
           self.add_metric(lowerCAmelCase_	,						name=self.name	,						aggregation="mean" if return_mean else ""  )

       return out





'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowercase    :							Tuple      =					{
    "configuration_xlm": ["XLM_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLMConfig", "XLMOnnxConfig"],
    "tokenization_xlm": ["XLMTokenizer"],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase    :							int      =					[
					    "XLM_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "XLMForMultipleChoice",
					    "XLMForQuestionAnswering",
					    "XLMForQuestionAnsweringSimple",
					    "XLMForSequenceClassification",
					    "XLMForTokenClassification",
					    "XLMModel",
					    "XLMPreTrainedModel",
					    "XLMWithLMHeadModel",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase    :							Tuple      =					[
					    "TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TFXLMForMultipleChoice",
					    "TFXLMForQuestionAnsweringSimple",
					    "TFXLMForSequenceClassification",
					    "TFXLMForTokenClassification",
					    "TFXLMMainLayer",
					    "TFXLMModel",
					    "TFXLMPreTrainedModel",
					    "TFXLMWithLMHeadModel",
					]


if TYPE_CHECKING:
					from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
					from .tokenization_xlm import XLMTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_xlm import (
										    XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
										    XLMForMultipleChoice,
										    XLMForQuestionAnswering,
										    XLMForQuestionAnsweringSimple,
										    XLMForSequenceClassification,
										    XLMForTokenClassification,
										    XLMModel,
										    XLMPreTrainedModel,
										    XLMWithLMHeadModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_xlm import (
										    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFXLMForMultipleChoice,
										    TFXLMForQuestionAnsweringSimple,
										    TFXLMForSequenceClassification,
										    TFXLMForTokenClassification,
										    TFXLMMainLayer,
										    TFXLMModel,
										    TFXLMPreTrainedModel,
										    TFXLMWithLMHeadModel,
										)

else:
					import sys

					lowercase    :							Union[str, Any]      =					_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

