'''simple docstring'''




import tensorflow as tf

from ...tf_utils import shape_list




class     a__      (   tf.keras.layers.Layer ):






     def __init__( self		:		Tuple  ,				a		:		Any  ,				a		:		Any  ,				a		:		Optional[Any]  ,				a		:		Optional[int]  ,				a		:		str=1  ,				a		:		Optional[int]=False  ,				**a		:		str    ):





         """simple docstring"""



         super().__init__(**a    )

         __lowerCamelCase		=  vocab_size
         __lowerCamelCase		=  d_embed
         __lowerCamelCase		=  d_proj

         __lowerCamelCase		=  cutoffs + [vocab_size]
         __lowerCamelCase		=  [0] + self.cutoffs
         __lowerCamelCase		=  div_val

         __lowerCamelCase		=  self.cutoffs[0]
         __lowerCamelCase		=  len(self.cutoffs    ) - 1
         __lowerCamelCase		=  self.shortlist_size + self.n_clusters
         __lowerCamelCase		=  keep_order

         __lowerCamelCase		=  []
         __lowerCamelCase		=  []






     def    SCREAMING_SNAKE_CASE__   ( self		:		int  ,				a		:		int    ):





         """simple docstring"""



         if self.n_clusters > 0:
             __lowerCamelCase		=  self.add_weight(
                 shape=(self.n_clusters, self.d_embed)  ,				initializer='''zeros'''  ,				trainable=a  ,				name='''cluster_weight'''    )
             __lowerCamelCase		=  self.add_weight(
                 shape=(self.n_clusters,)  ,				initializer='''zeros'''  ,				trainable=a  ,				name='''cluster_bias'''    )

         if self.div_val == 1:
             for i in range(len(self.cutoffs    )    ):
                 if self.d_proj != self.d_embed:
                     __lowerCamelCase		=  self.add_weight(
                         shape=(self.d_embed, self.d_proj)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_projs_._{i}"""  ,				)
                     self.out_projs.append(a    )
                 else:
                     self.out_projs.append(a    )
                 __lowerCamelCase		=  self.add_weight(
                     shape=(self.vocab_size, self.d_embed)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_layers_._{i}_._weight"""  ,				)
                 __lowerCamelCase		=  self.add_weight(
                     shape=(self.vocab_size,)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_layers_._{i}_._bias"""  ,				)
                 self.out_layers.append((weight, bias)    )
         else:
             for i in range(len(self.cutoffs    )    ):
                 __lowerCamelCase	,  __lowerCamelCase		=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                 __lowerCamelCase		=  self.d_embed // (self.div_val**i)

                 __lowerCamelCase		=  self.add_weight(
                     shape=(d_emb_i, self.d_proj)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_projs_._{i}"""    )
                 self.out_projs.append(a    )
                 __lowerCamelCase		=  self.add_weight(
                     shape=(r_idx - l_idx, d_emb_i)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_layers_._{i}_._weight"""  ,				)
                 __lowerCamelCase		=  self.add_weight(
                     shape=(r_idx - l_idx,)  ,				initializer='''zeros'''  ,				trainable=a  ,				name=f"""out_layers_._{i}_._bias"""  ,				)
                 self.out_layers.append((weight, bias)    )
         super().build(a    )






     @staticmethod
     def    SCREAMING_SNAKE_CASE__   ( a		:		Optional[Any]  ,				a		:		Dict  ,				a		:		Optional[Any]  ,				a		:		List[str]=None    ):





         """simple docstring"""



         __lowerCamelCase		=  x
         if proj is not None:
             __lowerCamelCase		=  tf.einsum('''ibd,ed->ibe'''  ,				a  ,				a    )
         return tf.einsum('''ibd,nd->ibn'''  ,				a  ,				a    ) + b






     @staticmethod
     def    SCREAMING_SNAKE_CASE__   ( a		:		List[str]  ,				a		:		Tuple    ):





         """simple docstring"""



         __lowerCamelCase		=  shape_list(a    )
         __lowerCamelCase		=  tf.range(lp_size[0]  ,				dtype=target.dtype    )
         __lowerCamelCase		=  tf.stack([r, target]  ,				1    )
         return tf.gather_nd(a  ,				a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[Any]  ,				a		:		Optional[int]  ,				a		:		Dict  ,				a		:		int=True  ,				a		:		Dict=False    ):





         """simple docstring"""



         __lowerCamelCase		=  0
         if self.n_clusters == 0:
             __lowerCamelCase		=  self._logit(a  ,				self.out_layers[0][0]  ,				self.out_layers[0][1]  ,				self.out_projs[0]    )
             if target is not None:
                 __lowerCamelCase		=  tf.nn.sparse_softmax_cross_entropy_with_logits(labels=a  ,				logits=a    )
             __lowerCamelCase		=  tf.nn.log_softmax(a  ,				axis=-1    )
         else:
             __lowerCamelCase		=  shape_list(a    )
             __lowerCamelCase		=  []
             __lowerCamelCase		=  tf.zeros(hidden_sizes[:2]    )
             for i in range(len(self.cutoffs    )    ):
                 __lowerCamelCase	,  __lowerCamelCase		=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                 if target is not None:
                     __lowerCamelCase		=  (target >= l_idx) & (target < r_idx)
                     __lowerCamelCase		=  tf.where(a    )
                     __lowerCamelCase		=  tf.boolean_mask(a  ,				a    ) - l_idx

                 if self.div_val == 1:
                     __lowerCamelCase		=  self.out_layers[0][0][l_idx:r_idx]
                     __lowerCamelCase		=  self.out_layers[0][1][l_idx:r_idx]
                 else:
                     __lowerCamelCase		=  self.out_layers[i][0]
                     __lowerCamelCase		=  self.out_layers[i][1]

                 if i == 0:
                     __lowerCamelCase		=  tf.concat([cur_W, self.cluster_weight]  ,				0    )
                     __lowerCamelCase		=  tf.concat([cur_b, self.cluster_bias]  ,				0    )

                     __lowerCamelCase		=  self._logit(a  ,				a  ,				a  ,				self.out_projs[0]    )
                     __lowerCamelCase		=  tf.nn.log_softmax(a    )
                     out.append(head_logprob[..., : self.cutoffs[0]]    )
                     if target is not None:
                         __lowerCamelCase		=  tf.boolean_mask(a  ,				a    )
                         __lowerCamelCase		=  self._gather_logprob(a  ,				a    )
                 else:
                     __lowerCamelCase		=  self._logit(a  ,				a  ,				a  ,				self.out_projs[i]    )
                     __lowerCamelCase		=  tf.nn.log_softmax(a    )
                     __lowerCamelCase		=  self.cutoffs[0] + i - 1  # No probability for the head cluster
                     __lowerCamelCase		=  head_logprob[..., cluster_prob_idx, None] + tail_logprob
                     out.append(a    )
                     if target is not None:
                         __lowerCamelCase		=  tf.boolean_mask(a  ,				a    )
                         __lowerCamelCase		=  tf.boolean_mask(a  ,				a    )
                         __lowerCamelCase		=  self._gather_logprob(a  ,				a    )
                         cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                 if target is not None:
                     loss += tf.scatter_nd(a  ,				-cur_logprob  ,				shape_list(a    )    )
             __lowerCamelCase		=  tf.concat(a  ,				axis=-1    )

         if target is not None:
             if return_mean:
                 __lowerCamelCase		=  tf.reduce_mean(a    )
             # Add the training-time loss value to the layer using `self.add_loss()`.
             self.add_loss(a    )

             # Log the loss as a metric (we could log arbitrary metrics,
             # including different metrics for training and inference.
             self.add_metric(a  ,				name=self.name  ,				aggregation='''mean''' if return_mean else ''''''    )

         return out


"""simple docstring"""




UpperCAmelCase__      :    List[str]    							= [
    9_9_9,
    8_0_0,
    7_9_9,
    6_0_0,
    5_9_9,
    5_0_0,
    4_0_0,
    3_9_9,
    3_7_7,
    3_5_5,
    3_3_3,
    3_1_1,
    2_8_8,
    2_6_6,
    2_4_4,
    2_2_2,
    2_0_0,
    1_9_9,
    1_7_7,
    1_5_5,
    1_3_3,
    1_1_1,
    8_8,
    6_6,
    4_4,
    2_2,
    0,
]

UpperCAmelCase__      :    int    							= [
    9_9_9,
    9_7_6,
    9_5_2,
    9_2_8,
    9_0_5,
    8_8_2,
    8_5_8,
    8_5_7,
    8_1_0,
    7_6_2,
    7_1_5,
    7_1_4,
    5_7_2,
    4_2_9,
    4_2_8,
    2_8_6,
    2_8_5,
    2_3_8,
    1_9_0,
    1_4_3,
    1_4_2,
    1_1_8,
    9_5,
    7_1,
    4_7,
    2_4,
    0,
]

UpperCAmelCase__      :    int    							= [
    9_9_9,
    9_8_8,
    9_7_7,
    9_6_6,
    9_5_5,
    9_4_4,
    9_3_3,
    9_2_2,
    9_1_1,
    9_0_0,
    8_9_9,
    8_7_9,
    8_5_9,
    8_4_0,
    8_2_0,
    8_0_0,
    7_9_9,
    7_6_6,
    7_3_3,
    7_0_0,
    6_9_9,
    6_5_0,
    6_0_0,
    5_9_9,
    5_0_0,
    4_9_9,
    4_0_0,
    3_9_9,
    3_5_0,
    3_0_0,
    2_9_9,
    2_6_6,
    2_3_3,
    2_0_0,
    1_9_9,
    1_7_9,
    1_5_9,
    1_4_0,
    1_2_0,
    1_0_0,
    9_9,
    8_8,
    7_7,
    6_6,
    5_5,
    4_4,
    3_3,
    2_2,
    1_1,
    0,
]

UpperCAmelCase__      :    int    							= [
    9_9_9,
    9_9_5,
    9_9_2,
    9_8_9,
    9_8_5,
    9_8_1,
    9_7_8,
    9_7_5,
    9_7_1,
    9_6_7,
    9_6_4,
    9_6_1,
    9_5_7,
    9_5_6,
    9_5_1,
    9_4_7,
    9_4_2,
    9_3_7,
    9_3_3,
    9_2_8,
    9_2_3,
    9_1_9,
    9_1_4,
    9_1_3,
    9_0_8,
    9_0_3,
    8_9_7,
    8_9_2,
    8_8_7,
    8_8_1,
    8_7_6,
    8_7_1,
    8_7_0,
    8_6_4,
    8_5_8,
    8_5_2,
    8_4_6,
    8_4_0,
    8_3_4,
    8_2_8,
    8_2_7,
    8_2_0,
    8_1_3,
    8_0_6,
    7_9_9,
    7_9_2,
    7_8_5,
    7_8_4,
    7_7_7,
    7_7_0,
    7_6_3,
    7_5_6,
    7_4_9,
    7_4_2,
    7_4_1,
    7_3_3,
    7_2_4,
    7_1_6,
    7_0_7,
    6_9_9,
    6_9_8,
    6_8_8,
    6_7_7,
    6_6_6,
    6_5_6,
    6_5_5,
    6_4_5,
    6_3_4,
    6_2_3,
    6_1_3,
    6_1_2,
    5_9_8,
    5_8_4,
    5_7_0,
    5_6_9,
    5_5_5,
    5_4_1,
    5_2_7,
    5_2_6,
    5_0_5,
    4_8_4,
    4_8_3,
    4_6_2,
    4_4_0,
    4_3_9,
    3_9_6,
    3_9_5,
    3_5_2,
    3_5_1,
    3_0_8,
    3_0_7,
    2_6_4,
    2_6_3,
    2_2_0,
    2_1_9,
    1_7_6,
    1_3_2,
    8_8,
    4_4,
    0,
]

UpperCAmelCase__      :    Tuple    							= [
    9_9_9,
    9_9_7,
    9_9_5,
    9_9_2,
    9_9_0,
    9_8_8,
    9_8_6,
    9_8_4,
    9_8_1,
    9_7_9,
    9_7_7,
    9_7_5,
    9_7_2,
    9_7_0,
    9_6_8,
    9_6_6,
    9_6_4,
    9_6_1,
    9_5_9,
    9_5_7,
    9_5_6,
    9_5_4,
    9_5_1,
    9_4_9,
    9_4_6,
    9_4_4,
    9_4_1,
    9_3_9,
    9_3_6,
    9_3_4,
    9_3_1,
    9_2_9,
    9_2_6,
    9_2_4,
    9_2_1,
    9_1_9,
    9_1_6,
    9_1_4,
    9_1_3,
    9_1_0,
    9_0_7,
    9_0_5,
    9_0_2,
    8_9_9,
    8_9_6,
    8_9_3,
    8_9_1,
    8_8_8,
    8_8_5,
    8_8_2,
    8_7_9,
    8_7_7,
    8_7_4,
    8_7_1,
    8_7_0,
    8_6_7,
    8_6_4,
    8_6_1,
    8_5_8,
    8_5_5,
    8_5_2,
    8_4_9,
    8_4_6,
    8_4_3,
    8_4_0,
    8_3_7,
    8_3_4,
    8_3_1,
    8_2_8,
    8_2_7,
    8_2_4,
    8_2_1,
    8_1_7,
    8_1_4,
    8_1_1,
    8_0_8,
    8_0_4,
    8_0_1,
    7_9_8,
    7_9_5,
    7_9_1,
    7_8_8,
    7_8_5,
    7_8_4,
    7_8_0,
    7_7_7,
    7_7_4,
    7_7_0,
    7_6_6,
    7_6_3,
    7_6_0,
    7_5_6,
    7_5_2,
    7_4_9,
    7_4_6,
    7_4_2,
    7_4_1,
    7_3_7,
    7_3_3,
    7_3_0,
    7_2_6,
    7_2_2,
    7_1_8,
    7_1_4,
    7_1_0,
    7_0_7,
    7_0_3,
    6_9_9,
    6_9_8,
    6_9_4,
    6_9_0,
    6_8_5,
    6_8_1,
    6_7_7,
    6_7_3,
    6_6_9,
    6_6_4,
    6_6_0,
    6_5_6,
    6_5_5,
    6_5_0,
    6_4_6,
    6_4_1,
    6_3_6,
    6_3_2,
    6_2_7,
    6_2_2,
    6_1_8,
    6_1_3,
    6_1_2,
    6_0_7,
    6_0_2,
    5_9_6,
    5_9_1,
    5_8_6,
    5_8_0,
    5_7_5,
    5_7_0,
    5_6_9,
    5_6_3,
    5_5_7,
    5_5_1,
    5_4_5,
    5_3_9,
    5_3_3,
    5_2_7,
    5_2_6,
    5_1_9,
    5_1_2,
    5_0_5,
    4_9_8,
    4_9_1,
    4_8_4,
    4_8_3,
    4_7_4,
    4_6_6,
    4_5_7,
    4_4_9,
    4_4_0,
    4_3_9,
    4_2_8,
    4_1_8,
    4_0_7,
    3_9_6,
    3_9_5,
    3_8_1,
    3_6_6,
    3_5_2,
    3_5_1,
    3_3_0,
    3_0_8,
    3_0_7,
    2_8_6,
    2_6_4,
    2_6_3,
    2_4_2,
    2_2_0,
    2_1_9,
    1_7_6,
    1_7_5,
    1_3_2,
    1_3_1,
    8_8,
    4_4,
    0,
]

UpperCAmelCase__      :    Union[str, Any]    							= [
    9_9_9,
    9_9_1,
    9_8_2,
    9_7_4,
    9_6_6,
    9_5_8,
    9_5_0,
    9_4_1,
    9_3_3,
    9_2_5,
    9_1_6,
    9_0_8,
    9_0_0,
    8_9_9,
    8_7_4,
    8_5_0,
    8_2_5,
    8_0_0,
    7_9_9,
    7_0_0,
    6_0_0,
    5_0_0,
    4_0_0,
    3_0_0,
    2_0_0,
    1_0_0,
    0,
]

UpperCAmelCase__      :    str    							= [
    9_9_9,
    9_9_2,
    9_8_5,
    9_7_8,
    9_7_1,
    9_6_4,
    9_5_7,
    9_4_9,
    9_4_2,
    9_3_5,
    9_2_8,
    9_2_1,
    9_1_4,
    9_0_7,
    9_0_0,
    8_9_9,
    8_7_9,
    8_5_9,
    8_4_0,
    8_2_0,
    8_0_0,
    7_9_9,
    7_6_6,
    7_3_3,
    7_0_0,
    6_9_9,
    6_5_0,
    6_0_0,
    5_9_9,
    5_0_0,
    4_9_9,
    4_0_0,
    3_9_9,
    3_0_0,
    2_9_9,
    2_0_0,
    1_9_9,
    1_0_0,
    9_9,
    0,
]

UpperCAmelCase__      :    str    							= [
    9_9_9,
    9_9_6,
    9_9_2,
    9_8_9,
    9_8_5,
    9_8_2,
    9_7_9,
    9_7_5,
    9_7_2,
    9_6_8,
    9_6_5,
    9_6_1,
    9_5_8,
    9_5_5,
    9_5_1,
    9_4_8,
    9_4_4,
    9_4_1,
    9_3_8,
    9_3_4,
    9_3_1,
    9_2_7,
    9_2_4,
    9_2_0,
    9_1_7,
    9_1_4,
    9_1_0,
    9_0_7,
    9_0_3,
    9_0_0,
    8_9_9,
    8_9_1,
    8_8_4,
    8_7_6,
    8_6_9,
    8_6_1,
    8_5_3,
    8_4_6,
    8_3_8,
    8_3_0,
    8_2_3,
    8_1_5,
    8_0_8,
    8_0_0,
    7_9_9,
    7_8_8,
    7_7_7,
    7_6_6,
    7_5_5,
    7_4_4,
    7_3_3,
    7_2_2,
    7_1_1,
    7_0_0,
    6_9_9,
    6_8_8,
    6_7_7,
    6_6_6,
    6_5_5,
    6_4_4,
    6_3_3,
    6_2_2,
    6_1_1,
    6_0_0,
    5_9_9,
    5_8_5,
    5_7_1,
    5_5_7,
    5_4_2,
    5_2_8,
    5_1_4,
    5_0_0,
    4_9_9,
    4_8_5,
    4_7_1,
    4_5_7,
    4_4_2,
    4_2_8,
    4_1_4,
    4_0_0,
    3_9_9,
    3_7_9,
    3_5_9,
    3_4_0,
    3_2_0,
    3_0_0,
    2_9_9,
    2_7_9,
    2_5_9,
    2_4_0,
    2_2_0,
    2_0_0,
    1_9_9,
    1_6_6,
    1_3_3,
    1_0_0,
    9_9,
    6_6,
    3_3,
    0,
]




