




import importlib
import math
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Tuple, Union

import flax
import jax.numpy as jnp

from ..utils import BaseOutput


snake_case_ : Optional[Any]    		=       "scheduler_config.json"







class __snake_case			(			a		):
   UpperCAmelCase__  :    List[str]          =  1
   UpperCAmelCase__  :    str          =  2
   UpperCAmelCase__  :    Tuple          =  3
   UpperCAmelCase__  :    Union[str, Any]          =  4
   UpperCAmelCase__  :    Tuple          =  5







@dataclass
class __snake_case			(			a		):
   UpperCAmelCase__  :    jnp.ndarray







class __snake_case			:
   UpperCAmelCase__  :    Tuple          =  SCHEDULER_CONFIG_NAME
   UpperCAmelCase__  :    Tuple          =  ['''dtype''']
   UpperCAmelCase__  :    List[Any]          =  []
   UpperCAmelCase__  :    List[str]          =  True







   @classmethod
   def lowerCamelCase						(      cls		:						int					,						_snake_case		:						Dict[str, Any] = None					,						_snake_case		:						Optional[str] = None					,						_snake_case		:						Any=False					,						**_snake_case		:						List[Any]					,						):

        """simple docstring"""


        UpperCAmelCase_     ,       UpperCAmelCase_          =							cls.load_config(
            pretrained_model_name_or_path=_snake_case					,						subfolder=_snake_case					,						return_unused_kwargs=_snake_case					,						**_snake_case					,						)
        UpperCAmelCase_     ,       UpperCAmelCase_          =							cls.from_config(_snake_case					,						return_unused_kwargs=_snake_case					,						**_snake_case)

        if hasattr(_snake_case					,						'''create_state''') and getattr(_snake_case					,						'''has_state'''					,						_snake_case):
             UpperCAmelCase_          =							scheduler.create_state()

        if return_unused_kwargs:
             return scheduler, state, unused_kwargs

        return scheduler, state







   def lowerCamelCase						(      self		:						Tuple					,						_snake_case		:						Union[str, os.PathLike]					,						_snake_case		:						bool = False					,						**_snake_case		:						str):

        """simple docstring"""


        self.save_config(save_directory=_snake_case					,						push_to_hub=_snake_case					,						**_snake_case)







   @property
   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        return self._get_compatibles()







   @classmethod
   def lowerCamelCase						(      cls		:						Optional[int]):

        """simple docstring"""


        UpperCAmelCase_          =							list(set([cls.__name__] + cls._compatibles))
        UpperCAmelCase_          =							importlib.import_module(__name__.split('''.''')[0])
        UpperCAmelCase_          =							[
            getattr(_snake_case					,						_snake_case) for c in compatible_classes_str if hasattr(_snake_case					,						_snake_case)
        ]
        return compatible_classes







def    A						(__A   :    jnp.ndarray   ,				__A   :    Tuple[int]       )   ->      jnp.ndarray:







     """simple docstring"""


     assert len(__A       ) >= x.ndim
     return jnp.broadcast_to(x.reshape(x.shape + (1,) * (len(__A       ) - x.ndim)       )   ,				__A       )


def    A						(__A   :    int   ,				__A   :    Tuple=0.999   ,				__A   :    int=jnp.floataa       )   ->      jnp.ndarray:







     """simple docstring"""


     def alpha_bar(__A   :    Optional[int]       ):
          return math.cos((time_step + 0.008) / 1.008 * math.pi / 2       ) ** 2

     UpperCAmelCase_          =							[]
     for i in range(__A       ):
          UpperCAmelCase_          =							i / num_diffusion_timesteps
          UpperCAmelCase_          =							(i + 1) / num_diffusion_timesteps
          betas.append(min(1 - alpha_bar(__A       ) / alpha_bar(__A       )   ,				__A       )       )
     return jnp.array(__A   ,				dtype=__A       )







@flax.struct.dataclass
class __snake_case			:
   UpperCAmelCase__  :    jnp.ndarray
   UpperCAmelCase__  :    jnp.ndarray
   UpperCAmelCase__  :    jnp.ndarray







   @classmethod
   def lowerCamelCase						(      cls		:						Any					,						_snake_case		:						Dict):

        """simple docstring"""


        UpperCAmelCase_          =							scheduler.config

        if config.trained_betas is not None:
             UpperCAmelCase_          =							jnp.asarray(config.trained_betas					,						dtype=scheduler.dtype)
        elif config.beta_schedule == "linear":
             UpperCAmelCase_          =							jnp.linspace(config.beta_start					,						config.beta_end					,						config.num_train_timesteps					,						dtype=scheduler.dtype)
        elif config.beta_schedule == "scaled_linear":
             # this schedule is very specific to the latent diffusion model.
             UpperCAmelCase_          =							(
                 jnp.linspace(
                     config.beta_start**0.5					,						config.beta_end**0.5					,						config.num_train_timesteps					,						dtype=scheduler.dtype)
                 ** 2
             )
        elif config.beta_schedule == "squaredcos_cap_v2":
             # Glide cosine schedule
             UpperCAmelCase_          =							betas_for_alpha_bar(config.num_train_timesteps					,						dtype=scheduler.dtype)
        else:
             raise NotImplementedError(
                 F"""beta_schedule {config.beta_schedule} is not implemented for scheduler {scheduler.__class__.__name__}""")

        UpperCAmelCase_          =							1.0 - betas

        UpperCAmelCase_          =							jnp.cumprod(_snake_case					,						axis=0)

        return cls(
            alphas=_snake_case					,						betas=_snake_case					,						alphas_cumprod=_snake_case					,						)







def    A						(__A   :    CommonSchedulerState   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray       )   ->      Dict:







     """simple docstring"""


     UpperCAmelCase_          =							state.alphas_cumprod

     UpperCAmelCase_          =							alphas_cumprod[timesteps] ** 0.5
     UpperCAmelCase_          =							sqrt_alpha_prod.flatten()
     UpperCAmelCase_          =							broadcast_to_shape_from_left(__A   ,				original_samples.shape       )

     UpperCAmelCase_          =							(1 - alphas_cumprod[timesteps]) ** 0.5
     UpperCAmelCase_          =							sqrt_one_minus_alpha_prod.flatten()
     UpperCAmelCase_          =							broadcast_to_shape_from_left(__A   ,				original_samples.shape       )

     return sqrt_alpha_prod, sqrt_one_minus_alpha_prod


def    A						(__A   :    CommonSchedulerState   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray       )   ->      Dict:







     """simple docstring"""


     UpperCAmelCase_     ,       UpperCAmelCase_          =							get_sqrt_alpha_prod(__A   ,				__A   ,				__A   ,				__A       )
     UpperCAmelCase_          =							sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
     return noisy_samples


def    A						(__A   :    CommonSchedulerState   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray   ,				__A   :    jnp.ndarray       )   ->      Any:







     """simple docstring"""


     UpperCAmelCase_     ,       UpperCAmelCase_          =							get_sqrt_alpha_prod(__A   ,				__A   ,				__A   ,				__A       )
     UpperCAmelCase_          =							sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
     return velocity












import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

snake_case_ : Tuple    		=       logging.get_logger(__name__)


def    A						(__A   :    bool   ,				__A   :    bool       )   ->      Optional[Any]:







     """simple docstring"""


     def run_func(__A   :    Optional[Any]       ):
          @wraps(__A       )
          def run_in_eager_mode(*__A   :    Dict   ,				**__A   :    List[Any]       ):
               return func(*__A   ,				**__A       )

          @wraps(__A       )
          @tf.function(experimental_compile=__A       )
          def run_in_graph_mode(*__A   :    Optional[Any]   ,				**__A   :    Any       ):
               return func(*__A   ,				**__A       )

          if do_eager_mode is True:
               if use_xla is not False:
                    raise ValueError(
                        '''Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'''       )
               return run_in_eager_mode
          else:
               return run_in_graph_mode

     return run_func


def    A						(__A   :    int   ,				__A   :    int   ,				__A   :    int       )   ->      ["tf.Tensor"]:







     """simple docstring"""


     UpperCAmelCase_          =							random.Random()
     UpperCAmelCase_          =							[rng.randint(0   ,				vocab_size - 1       ) for i in range(batch_size * sequence_length       )]
     return tf.constant(__A   ,				shape=(batch_size, sequence_length)   ,				dtype=tf.intaa       )







class __snake_case			(			a		):
   UpperCAmelCase__  :    TensorFlowBenchmarkArguments
   UpperCAmelCase__  :    PretrainedConfig
   UpperCAmelCase__  :    str          =  "TensorFlow"







   @property
   def lowerCamelCase						(      self		:						List[str]):

        """simple docstring"""


        return tf.__version__







   def lowerCamelCase						(      self		:						Dict					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_inference_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_speed(_inference)







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_train_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_speed(_train)







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,						_snake_case)
        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')
        UpperCAmelCase_          =							self._prepare_inference_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_memory(_inference)







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]					,						_snake_case)
        UpperCAmelCase_          =							self.args.strategy
        if strategy is None:
             raise ValueError('''A device strategy has to be initialized before using TensorFlow.''')

        UpperCAmelCase_          =							self._prepare_train_func(_snake_case					,						_snake_case					,						_snake_case)
        return self._measure_memory(_train)







   def lowerCamelCase						(      self		:						Optional[int]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.config_dict[model_name]

        if self.args.fpaa:
             raise NotImplementedError('''Mixed precision is currently not supported.''')

        UpperCAmelCase_          =							(
            hasattr(_snake_case					,						'''architectures''')
            and isinstance(config.architectures					,						_snake_case)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                  UpperCAmelCase_          =							'''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
                  UpperCAmelCase_          =							__import__('''transformers'''					,						fromlist=[model_class])
                  UpperCAmelCase_          =							getattr(_snake_case					,						_snake_case)
                  UpperCAmelCase_          =							model_cls(_snake_case)
             except ImportError:
                  raise ImportError(
                      F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                      ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''')
        else:
             UpperCAmelCase_          =							TF_MODEL_MAPPING[config.__class__](_snake_case)

        # encoder-decoder has vocab size saved differently
        UpperCAmelCase_          =							config.vocab_size if hasattr(_snake_case					,						'''vocab_size''') else config.encoder.vocab_size
        UpperCAmelCase_          =							random_input_ids(_snake_case					,						_snake_case					,						_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_decoder_forward():
             return model(_snake_case					,						decoder_input_ids=_snake_case					,						training=_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_forward():
             return model(_snake_case					,						training=_snake_case)

        UpperCAmelCase_          =							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference







   def lowerCamelCase						(      self		:						Optional[Any]					,						_snake_case		:						str					,						_snake_case		:						int					,						_snake_case		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.config_dict[model_name]

        if self.args.eager_mode is not False:
             raise ValueError('''Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.''')

        if self.args.fpaa:
             raise NotImplementedError('''Mixed precision is currently not supported.''')

        UpperCAmelCase_          =							(
            hasattr(_snake_case					,						'''architectures''')
            and isinstance(config.architectures					,						_snake_case)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                  UpperCAmelCase_          =							'''TF''' + config.architectures[0]  # prepend 'TF' for tensorflow model
                  UpperCAmelCase_          =							__import__('''transformers'''					,						fromlist=[model_class])
                  UpperCAmelCase_          =							getattr(_snake_case					,						_snake_case)
                  UpperCAmelCase_          =							model_cls(_snake_case)
             except ImportError:
                  raise ImportError(
                      F"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
                      ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.''')
        else:
             UpperCAmelCase_          =							TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_snake_case)

        # encoder-decoder has vocab size saved differently
        UpperCAmelCase_          =							config.vocab_size if hasattr(_snake_case					,						'''vocab_size''') else config.encoder.vocab_size
        UpperCAmelCase_          =							random_input_ids(_snake_case					,						_snake_case					,						_snake_case)

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_decoder_train():
             UpperCAmelCase_          =							model(_snake_case					,						decoder_input_ids=_snake_case					,						labels=_snake_case					,						training=_snake_case)[0]
             UpperCAmelCase_          =							tf.gradients(_snake_case					,						model.trainable_variables)
             return gradients

        @run_with_tf_optimizations(self.args.eager_mode					,						self.args.use_xla)
        def encoder_train():
             UpperCAmelCase_          =							model(_snake_case					,						labels=_snake_case					,						training=_snake_case)[0]
             UpperCAmelCase_          =							tf.gradients(_snake_case					,						model.trainable_variables)
             return gradients

        UpperCAmelCase_          =							encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train







   def lowerCamelCase						(      self		:						Any					,						_snake_case		:						Optional[Any]):

        """simple docstring"""


        with self.args.strategy.scope():
             try:
                  if self.args.is_tpu or self.args.use_xla:
                       # run additional 10 times to stabilize compilation for tpu
                       logger.info('''Do inference on TPU. Running model 5 times to stabilize compilation''')
                       timeit.repeat(_snake_case					,						repeat=1					,						number=5)

                  # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                  UpperCAmelCase_          =							timeit.repeat(
                      _snake_case					,						repeat=self.args.repeat					,						number=10					,						)

                  return min(_snake_case) / 1_0.0
             except ResourceExhaustedError as e:
                  self.print_fn(F"""Doesn't fit on GPU. {e}""")







   def lowerCamelCase						(      self		:						Dict					,						_snake_case		:						Callable[[], None]):

        """simple docstring"""


        logger.info(
            '''Note that TensorFlow allocates more memory than '''
            '''it might need to speed up computation. '''
            '''The memory reported here corresponds to the memory '''
            '''reported by `nvidia-smi`, which can vary depending '''
            '''on total available memory on the GPU that is used.''')
        with self.args.strategy.scope():
             try:
                  if self.args.trace_memory_line_by_line:
                       if not self.args.eager_mode:
                            raise ValueError(
                                '''`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'''
                                ''' consumption line by line.''')
                       UpperCAmelCase_          =							start_memory_tracing('''transformers''')

                  if self.args.is_tpu:
                       # tpu
                       raise NotImplementedError(
                           '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'''
                           ''' with `args.memory=False`''')
                  elif self.args.is_gpu:
                       # gpu
                       if not is_pyanvml_available():
                            logger.warning(
                                '''py3nvml not installed, we won\'t log GPU memory usage. '''
                                '''Install py3nvml (pip install py3nvml) to log information about GPU.''')
                            UpperCAmelCase_          =							'''N/A'''
                       else:
                            logger.info(
                                '''Measuring total GPU usage on GPU device. Make sure to not have additional processes'''
                                ''' running on the same GPU.''')
                            # init nvml
                            nvml.nvmlInit()
                            func()
                            UpperCAmelCase_          =							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
                            UpperCAmelCase_          =							nvml.nvmlDeviceGetMemoryInfo(_snake_case)
                            UpperCAmelCase_          =							meminfo.used
                            UpperCAmelCase_          =							Memory(_snake_case)
                            # shutdown nvml
                            nvml.nvmlShutdown()
                  else:
                       # cpu
                       if self.args.trace_memory_line_by_line:
                            logger.info(
                                '''When enabling line by line tracing, the max peak memory for CPU is inaccurate in'''
                                ''' TensorFlow.''')
                            UpperCAmelCase_          =							None
                       else:
                            UpperCAmelCase_          =							measure_peak_memory_cpu(_snake_case)
                            UpperCAmelCase_          =							Memory(_snake_case) if isinstance(_snake_case					,						_snake_case) else memory_bytes
                  if self.args.trace_memory_line_by_line:
                       UpperCAmelCase_          =							stop_memory_tracing(_snake_case)
                       if memory is None:
                            UpperCAmelCase_          =							summary.total
                  else:
                       UpperCAmelCase_          =							None

                  return memory, summary
             except ResourceExhaustedError as e:
                  self.print_fn(F"""Doesn't fit on GPU. {e}""")
                  return "N/A", None






