

from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMModel,
        TFXLMWithLMHeadModel,
        XLMConfig,
    )







class 					__lowerCamelCase							:







      def __init__(	self				,    lowerCamelCase				,    )		->			str:
        snake_case_					= parent
        snake_case_					= 13
        snake_case_					= 7
        snake_case_					= True
        snake_case_					= True
        snake_case_					= True
        snake_case_					= True
        snake_case_					= True
        snake_case_					= False
        snake_case_					= False
        snake_case_					= False
        snake_case_					= 2
        snake_case_					= 99
        snake_case_					= 0
        snake_case_					= 32
        snake_case_					= 2
        snake_case_					= 4
        snake_case_					= 0.1
        snake_case_					= 0.1
        snake_case_					= 512
        snake_case_					= 16
        snake_case_					= 2
        snake_case_					= 0.02
        snake_case_					= 3
        snake_case_					= 4
        snake_case_					= """last"""
        snake_case_					= True
        snake_case_					= None
        snake_case_					= 0







      def       lowerCAmelCase_       (	self  )		->			Dict:
        snake_case_					= ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size  )
        snake_case_					= random_attention_mask([self.batch_size, self.seq_length]				,    dtype=tf.floataa  )

        snake_case_					= None
        if self.use_input_lengths:
          snake_case_					= (
              ids_tensor([self.batch_size]				,    vocab_size=2  ) + self.seq_length - 2
          )  # small variation of seq_length

        snake_case_					= None
        if self.use_token_type_ids:
          snake_case_					= ids_tensor([self.batch_size, self.seq_length]				,    self.n_langs  )

        snake_case_					= None
        snake_case_					= None
        snake_case_					= None
        if self.use_labels:
          snake_case_					= ids_tensor([self.batch_size]				,    self.type_sequence_label_size  )
          snake_case_					= ids_tensor([self.batch_size, self.seq_length]				,    self.num_labels  )
          snake_case_					= ids_tensor([self.batch_size]				,    2				,    dtype=tf.floataa  )
          snake_case_					= ids_tensor([self.batch_size]				,    self.num_choices  )

        snake_case_					= XLMConfig(
            vocab_size=self.vocab_size				,    n_special=self.n_special				,    emb_dim=self.hidden_size				,    n_layers=self.num_hidden_layers				,    n_heads=self.num_attention_heads				,    dropout=self.hidden_dropout_prob				,    attention_dropout=self.attention_probs_dropout_prob				,    gelu_activation=self.gelu_activation				,    sinusoidal_embeddings=self.sinusoidal_embeddings				,    asm=self.asm				,    causal=self.causal				,    n_langs=self.n_langs				,    max_position_embeddings=self.max_position_embeddings				,    initializer_range=self.initializer_range				,    summary_type=self.summary_type				,    use_proj=self.use_proj				,    bos_token_id=self.bos_token_id				,    )

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			int:
        snake_case_					= TFXLMModel(config=lowerCamelCase  )
        snake_case_					= {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
        snake_case_					= model(lowerCamelCase  )

        snake_case_					= [input_ids, input_mask]
        snake_case_					= model(lowerCamelCase  )
        self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size)  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			Union[str, Any]:
        snake_case_					= TFXLMWithLMHeadModel(lowerCamelCase  )

        snake_case_					= {"""input_ids""": input_ids, """lengths""": input_lengths, """langs""": token_type_ids}
        snake_case_					= model(lowerCamelCase  )

        snake_case_					= outputs

        self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.vocab_size)  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			int:
        snake_case_					= TFXLMForQuestionAnsweringSimple(lowerCamelCase  )

        snake_case_					= {"""input_ids""": input_ids, """lengths""": input_lengths}

        snake_case_					= model(lowerCamelCase  )

        self.parent.assertEqual(result.start_logits.shape				,    (self.batch_size, self.seq_length)  )
        self.parent.assertEqual(result.end_logits.shape				,    (self.batch_size, self.seq_length)  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			Union[str, Any]:
        snake_case_					= TFXLMForSequenceClassification(lowerCamelCase  )

        snake_case_					= {"""input_ids""": input_ids, """lengths""": input_lengths}

        snake_case_					= model(lowerCamelCase  )

        self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.type_sequence_label_size)  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			Any:
        snake_case_					= self.num_labels
        snake_case_					= TFXLMForTokenClassification(config=lowerCamelCase  )
        snake_case_					= {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
        snake_case_					= model(lowerCamelCase  )
        self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.num_labels)  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			Tuple:
        snake_case_					= self.num_choices
        snake_case_					= TFXLMForMultipleChoice(config=lowerCamelCase  )
        snake_case_					= tf.tile(tf.expand_dims(lowerCamelCase				,    1  )				,    (1, self.num_choices, 1)  )
        snake_case_					= tf.tile(tf.expand_dims(lowerCamelCase				,    1  )				,    (1, self.num_choices, 1)  )
        snake_case_					= tf.tile(tf.expand_dims(lowerCamelCase				,    1  )				,    (1, self.num_choices, 1)  )
        snake_case_					= {
            """input_ids""": multiple_choice_inputs_ids,
            """attention_mask""": multiple_choice_input_mask,
            """token_type_ids""": multiple_choice_token_type_ids,
        }
        snake_case_					= model(lowerCamelCase  )
        self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.num_choices)  )







      def       lowerCAmelCase_       (	self  )		->			List[Any]:
        snake_case_					= self.prepare_config_and_inputs()
        (
            (
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							(
            snake_case_
        )		,							
        )					= config_and_inputs
        snake_case_					= {
            """input_ids""": input_ids,
            """token_type_ids""": token_type_ids,
            """langs""": token_type_ids,
            """lengths""": input_lengths,
        }
        return config, inputs_dict







@require_tf
class 					__lowerCamelCase							(      __snake_case    ,				__snake_case    ,				unittest.TestCase      ):
      lowerCamelCase_      : Tuple  			=   (
          (
              TFXLMModel,
              TFXLMWithLMHeadModel,
              TFXLMForSequenceClassification,
              TFXLMForQuestionAnsweringSimple,
              TFXLMForTokenClassification,
              TFXLMForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      lowerCamelCase_      : Dict  			=   (
          (TFXLMWithLMHeadModel,) if is_tf_available() else ()
      )  # TODO (PVP): Check other models whether language generation is also applicable
      lowerCamelCase_      : Tuple  			=   (
          {
              'feature-extraction': TFXLMModel,
              'fill-mask': TFXLMWithLMHeadModel,
              'question-answering': TFXLMForQuestionAnsweringSimple,
              'text-classification': TFXLMForSequenceClassification,
              'text-generation': TFXLMWithLMHeadModel,
              'token-classification': TFXLMForTokenClassification,
              'zero-shot': TFXLMForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      lowerCamelCase_      : int  			=   False
      lowerCamelCase_      : Tuple  			=   False







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  )		->			Union[str, Any]:
        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith("""Fast"""  )
        ):
          # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
          # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
          # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
          return True

        return False







      def       lowerCAmelCase_       (	self  )		->			Union[str, Any]:
        snake_case_					= TFXLMModelTester(self  )
        snake_case_					= ConfigTester(self				,    config_class=lowerCamelCase				,    emb_dim=37  )







      def       lowerCAmelCase_       (	self  )		->			List[Any]:
        self.config_tester.run_common_tests()







      def       lowerCAmelCase_       (	self  )		->			Optional[Any]:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_model(*lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			Tuple:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_lm_head(*lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			List[str]:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_qa(*lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			int:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_sequence_classif(*lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			Tuple:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_for_token_classification(*lowerCamelCase  )







      def       lowerCAmelCase_       (	self  )		->			Optional[int]:
        snake_case_					= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_for_multiple_choice(*lowerCamelCase  )







      @slow
      def       lowerCAmelCase_       (	self  )		->			int:
        for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          snake_case_					= TFXLMModel.from_pretrained(lowerCamelCase  )
          self.assertIsNotNone(lowerCamelCase  )







@require_tf
class 					__lowerCamelCase							(      unittest.TestCase      ):







      @slow
      def       lowerCAmelCase_       (	self  )		->			Optional[Any]:
        snake_case_					= TFXLMWithLMHeadModel.from_pretrained("""xlm-mlm-en-2048"""  )
        snake_case_					= tf.convert_to_tensor([[14, 447]]				,    dtype=tf.intaa  )  # the president
        snake_case_					= [
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
        ]  # the president the president the president the president the president the president the president the president the president the president
        # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
        snake_case_					= model.generate(lowerCamelCase				,    do_sample=lowerCamelCase  )
        self.assertListEqual(output_ids[0].numpy().tolist()				,    lowerCamelCase  )


from dataclasses import dataclass
from typing import Optional, Tuple, Union

import flax
import jax.numpy as jnp
from scipy import integrate

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils_flax import (
    CommonSchedulerState,
    FlaxKarrasDiffusionSchedulers,
    FlaxSchedulerMixin,
    FlaxSchedulerOutput,
    broadcast_to_shape_from_left,
)







@flax.struct.dataclass
class 					__lowerCamelCase							:
      lowerCamelCase_      : CommonSchedulerState

      # setable values
      lowerCamelCase_      : jnp.ndarray
      lowerCamelCase_      : jnp.ndarray
      lowerCamelCase_      : jnp.ndarray
      lowerCamelCase_      : Optional[int]  			=   None

      # running values
      lowerCamelCase_      : Optional[jnp.ndarray]  			=   None







      @classmethod
      def       lowerCAmelCase_       (	cls				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  )		->			Union[str, Any]:
        return cls(common=lowerCamelCase				,    init_noise_sigma=lowerCamelCase				,    timesteps=lowerCamelCase				,    sigmas=lowerCamelCase  )







@dataclass
class 					__lowerCamelCase							(      __snake_case      ):
      lowerCamelCase_      : LMSDiscreteSchedulerState







class 					__lowerCamelCase							(      __snake_case    ,				__snake_case      ):
      lowerCamelCase_      : str  			=   [e.name for e in FlaxKarrasDiffusionSchedulers]

      lowerCamelCase_      : jnp.dtype







      @property
      def       lowerCAmelCase_       (	self  )		->			List[str]:
        return True







      @register_to_config
      def __init__(	self				,    lowerCamelCase = 1000				,    lowerCamelCase = 0.0001				,    lowerCamelCase = 0.02				,    lowerCamelCase = "linear"				,    lowerCamelCase = None				,    lowerCamelCase = "epsilon"				,    lowerCamelCase = jnp.floataa				,    )		->			Optional[Any]:
        snake_case_					= dtype







      def       lowerCAmelCase_       (	self				,    lowerCamelCase = None  )		->			LMSDiscreteSchedulerState:
        if common is None:
          snake_case_					= CommonSchedulerState.create(self  )

        snake_case_					= jnp.arange(0				,    self.config.num_train_timesteps  ).round()[::-1]
        snake_case_					= ((1 - common.alphas_cumprod) / common.alphas_cumprod) ** 0.5

        # standard deviation of the initial noise distribution
        snake_case_					= sigmas.max()

        return LMSDiscreteSchedulerState.create(
            common=lowerCamelCase				,    init_noise_sigma=lowerCamelCase				,    timesteps=lowerCamelCase				,    sigmas=lowerCamelCase				,    )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  )		->			jnp.ndarray:
        ((snake_case_)		,							)					= jnp.where(state.timesteps == timestep				,    size=1  )
        snake_case_					= step_index[0]

        snake_case_					= state.sigmas[step_index]
        snake_case_					= sample / ((sigma**2 + 1) ** 0.5)
        return sample







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  )		->			Any:

        def lms_derivative(lowerCamelCase  ):
          snake_case_					= 1.0
          for k in range(lowerCamelCase  ):
            if current_order == k:
              continue
            prod *= (tau - state.sigmas[t - k]) / (state.sigmas[t - current_order] - state.sigmas[t - k])
          return prod

        snake_case_					= integrate.quad(lowerCamelCase				,    state.sigmas[t]				,    state.sigmas[t + 1]				,    epsrel=1e-4  )[0]

        return integrated_coeff







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase = ()  )		->			LMSDiscreteSchedulerState:
        snake_case_					= jnp.linspace(self.config.num_train_timesteps - 1				,    0				,    lowerCamelCase				,    dtype=self.dtype  )

        snake_case_					= jnp.floor(lowerCamelCase  ).astype(jnp.intaa  )
        snake_case_					= jnp.ceil(lowerCamelCase  ).astype(jnp.intaa  )

        snake_case_					= jnp.mod(lowerCamelCase				,    1.0  )

        snake_case_					= ((1 - state.common.alphas_cumprod) / state.common.alphas_cumprod) ** 0.5
        snake_case_					= (1 - frac) * sigmas[low_idx] + frac * sigmas[high_idx]
        snake_case_					= jnp.concatenate([sigmas, jnp.array([0.0]				,    dtype=self.dtype  )]  )

        snake_case_					= timesteps.astype(jnp.intaa  )

        # initial running values
        snake_case_					= jnp.zeros((0,) + shape				,    dtype=self.dtype  )

        return state.replace(
            timesteps=lowerCamelCase				,    sigmas=lowerCamelCase				,    num_inference_steps=lowerCamelCase				,    derivatives=lowerCamelCase				,    )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase = 4				,    lowerCamelCase = True				,    )		->			Union[FlaxLMSSchedulerOutput, Tuple]:
        if state.num_inference_steps is None:
          raise ValueError(
              """Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"""  )

        snake_case_					= state.sigmas[timestep]

        # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
        if self.config.prediction_type == "epsilon":
          snake_case_					= sample - sigma * model_output
        elif self.config.prediction_type == "v_prediction":
          # * c_out + input * c_skip
          snake_case_					= model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1))
        else:
          raise ValueError(
              f'''prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`'''  )

        # 2. Convert to an ODE derivative
        snake_case_					= (sample - pred_original_sample) / sigma
        snake_case_					= state.replace(derivatives=jnp.append(state.derivatives				,    lowerCamelCase  )  )
        if len(state.derivatives  ) > order:
          snake_case_					= state.replace(derivatives=jnp.delete(state.derivatives				,    0  )  )

        # 3. Compute linear multistep coefficients
        snake_case_					= min(timestep + 1				,    lowerCamelCase  )
        snake_case_					= [self.get_lms_coefficient(lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase  ) for curr_order in range(lowerCamelCase  )]

        # 4. Compute previous sample based on the derivatives path
        snake_case_					= sample + sum(
            coeff * derivative for coeff, derivative in zip(lowerCamelCase				,    reversed(state.derivatives  )  )  )

        if not return_dict:
          return (prev_sample, state)

        return FlaxLMSSchedulerOutput(prev_sample=lowerCamelCase				,    state=lowerCamelCase  )







      def       lowerCAmelCase_       (	self				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    lowerCamelCase				,    )		->			jnp.ndarray:
        snake_case_					= state.sigmas[timesteps].flatten()
        snake_case_					= broadcast_to_shape_from_left(lowerCamelCase				,    noise.shape  )

        snake_case_					= original_samples + noise * sigma

        return noisy_samples







      def __len__(	self  )		->			Optional[Any]:
        return self.config.num_train_timesteps