
import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
  import torch_xla.core.xla_model as xm  # noqa: F401

  _UpperCAmelCase    			=					True
except ImportError:
  _UpperCAmelCase    			=					False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_UpperCAmelCase    			=					torch.distributed.is_available()


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						importlib.util.find_spec(_a  ) is not None
  if package_exists:
    try:
      _lowerCamelCase      =						importlib.metadata.metadata(_a  )
      return True
    except importlib.metadata.PackageNotFoundError:
      return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _torch_distributed_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  try:
    pass
  except ImportError:
    print(
        '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
        ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
        ''' Bindings for PyTorch*.'''  )
  return (
      importlib.util.find_spec('''torch_ccl'''  ) is not None
      or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''  ) is not None
  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return importlib.metadata.version('''oneccl_bind_pt'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformer_engine'''  )


@lru_cache
def 		_lowerCamelCase       (   _a=True  ):
  """simple docstring"""




  if torch.cuda.is_available():
    return False
  if _tpu_available and check_device:
    try:
      # Will raise a RuntimeError if no XLA configuration is found
      _lowerCamelCase      =						xm.xla_device()
      return True
    except RuntimeError:
      return False
  return _tpu_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''deepspeed'''  )


def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if is_tpu_available():
    return not ignore_tpu
  if torch.cuda.is_available():
    return torch.cuda.is_bfaa_supported()
  if is_npu_available():
    return False
  return True


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.39.0'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.37.2'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''bitsandbytes'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''			,   '''False'''  )  ) == 1:
    _lowerCamelCase      =						importlib.util.find_spec('''megatron'''  ) is not None
    if package_exists:
      try:
        _lowerCamelCase      =						parse(importlib.metadata.version('''megatron-lm'''  )  )
        return compare_versions(_a			,   '''>='''			,   '''2.2.0'''  )
      except Exception as e:
        warnings.warn(F'''Parse Megatron version failed. Exception:{e}'''  )
        return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''safetensors'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformers'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''datasets'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''aim'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tensorboard'''  ) or _is_package_available('''tensorboardX'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''wandb'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''comet_ml'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''boto3'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if _is_package_available('''rich'''  ):
    if "ACCELERATE_DISABLE_RICH" in os.environ:
      warnings.warn(
          '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''  )
      return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''			,   _a  )
    return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''			,   _a  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''sagemaker'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tqdm'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''mlflow'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return is_torch_version('''>='''			,   '''1.12'''  ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  def get_major_and_minor_from_version(_a  ):
    return str(version.parse(_a  ).major  ) + "." + str(version.parse(_a  ).minor  )

  _lowerCamelCase      =						importlib.metadata.version('''torch'''  )
  if importlib.util.find_spec('''intel_extension_for_pytorch'''  ) is None:
    return False
  _lowerCamelCase      =						'''N/A'''
  try:
    _lowerCamelCase      =						importlib.metadata.version('''intel_extension_for_pytorch'''  )
  except importlib.metadata.PackageNotFoundError:
    return False
  _lowerCamelCase      =						get_major_and_minor_from_version(_torch_version  )
  _lowerCamelCase      =						get_major_and_minor_from_version(_ipex_version  )
  if torch_major_and_minor != ipex_major_and_minor:
    warnings.warn(
        F'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
        F''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''  )
    return False
  return True


@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if importlib.util.find_spec('''torch'''  ) is None or importlib.util.find_spec('''torch_npu'''  ) is None:
    return False

  import torch
  import torch_npu  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no NPU is found
      _lowerCamelCase      =						torch.npu.device_count()
      return torch.npu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''npu'''  ) and torch.npu.is_available()






@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if not parse_flag_from_env('''ACCELERATE_USE_XPU'''			,   default=_a  ):
    return False
  if is_ipex_available():
    import torch

    if is_torch_version('''<='''			,   '''1.12'''  ):
      return False
  else:
    return False

  import intel_extension_for_pytorch  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no XPU  is found
      _lowerCamelCase      =						torch.xpu.device_count()
      return torch.xpu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''xpu'''  ) and torch.xpu.is_available()






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_UpperCAmelCase    			=					logging.get_logger(__name__)

_UpperCAmelCase    			=					{
    "gpt2": "https://huggingface.co/gpt2/resolve/main/config.json",
    "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/config.json",
    "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/config.json",
    "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/config.json",
    "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/config.json",
}

class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  _UpperCamelCase						 = "gpt2"
  _UpperCamelCase						 = ["past_key_values"]
  _UpperCamelCase						 = {
      "hidden_size": "n_embd",
      "max_position_embeddings": "n_positions",
      "num_attention_heads": "n_head",
      "num_hidden_layers": "n_layer",
  }


  def __init__(					self	,		a__=5_02_57	,		a__=10_24	,		a__=7_68	,		a__=12	,		a__=12	,		a__=None	,		a__="gelu_new"	,		a__=0.1	,		a__=0.1	,		a__=0.1	,		a__=1E-5	,		a__=0.02	,		a__="cls_index"	,		a__=True	,		a__=None	,		a__=True	,		a__=0.1	,		a__=True	,		a__=True	,		a__=5_02_56	,		a__=5_02_56	,		a__=False	,		a__=False	,		**a__	,		):
    _lowerCamelCase      =						vocab_size
    _lowerCamelCase      =						n_positions
    _lowerCamelCase      =						n_embd
    _lowerCamelCase      =						n_layer
    _lowerCamelCase      =						n_head
    _lowerCamelCase      =						n_inner
    _lowerCamelCase      =						activation_function
    _lowerCamelCase      =						resid_pdrop
    _lowerCamelCase      =						embd_pdrop
    _lowerCamelCase      =						attn_pdrop
    _lowerCamelCase      =						layer_norm_epsilon
    _lowerCamelCase      =						initializer_range
    _lowerCamelCase      =						summary_type
    _lowerCamelCase      =						summary_use_proj
    _lowerCamelCase      =						summary_activation
    _lowerCamelCase      =						summary_first_dropout
    _lowerCamelCase      =						summary_proj_to_labels
    _lowerCamelCase      =						scale_attn_weights
    _lowerCamelCase      =						use_cache
    _lowerCamelCase      =						scale_attn_by_inverse_layer_idx
    _lowerCamelCase      =						reorder_and_upcast_attn

    _lowerCamelCase      =						bos_token_id
    _lowerCamelCase      =						eos_token_id

    super().__init__(bos_token_id=a__	,		eos_token_id=a__	,		**a__		)







class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  def __init__(					self	,		a__	,		a__ = "default"	,		a__ = None	,		a__ = False	,		):
    super().__init__(a__	,		task=a__	,		patching_specs=a__	,		use_past=a__		)
    if not getattr(self._config	,		'''pad_token_id'''	,		a__		):
      # TODO: how to do that better?
      _lowerCamelCase      =						0


  @property
  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}		)
    if self.use_past:
      self.fill_with_past_key_values_(a__	,		direction='''inputs'''		)
      _lowerCamelCase      =						{0: '''batch''', 1: '''past_sequence + sequence'''}
    else:
      _lowerCamelCase      =						{0: '''batch''', 1: '''sequence'''}

    return common_inputs


  @property
  def     _UpperCAmelCase  (					self		):
    return self._config.n_layer


  @property
  def     _UpperCAmelCase  (					self		):
    return self._config.n_head


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = -1	,		a__ = -1	,		a__ = False	,		a__ = None	,		):
    _lowerCamelCase      =						super(a__	,		self		).generate_dummy_inputs(
        a__	,		batch_size=a__	,		seq_length=a__	,		is_pair=a__	,		framework=a__		)

    # We need to order the input in the way they appears in the forward()
    _lowerCamelCase      =						OrderedDict({'''input_ids''': common_inputs['''input_ids''']}		)

    # Need to add the past_keys
    if self.use_past:
      if not is_torch_available():
        raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''		)
      else:
        import torch

        _lowerCamelCase    ,    _lowerCamelCase      =						common_inputs['''input_ids'''].shape
        # Not using the same length for past_key_values
        _lowerCamelCase      =						seqlen + 2
        _lowerCamelCase      =						(
            batch,
            self.num_attention_heads,
            past_key_values_length,
            self._config.hidden_size // self.num_attention_heads,
        )
        _lowerCamelCase      =						[
            (torch.zeros(a__		), torch.zeros(a__		)) for _ in range(self.num_layers		)
        ]

    _lowerCamelCase      =						common_inputs['''attention_mask''']
    if self.use_past:
      _lowerCamelCase      =						ordered_inputs['''attention_mask'''].dtype
      _lowerCamelCase      =						torch.cat(
          [ordered_inputs['''attention_mask'''], torch.ones(a__	,		a__	,		dtype=a__		)]	,		dim=1		)

    return ordered_inputs


  @property
  def     _UpperCAmelCase  (					self		):
    return 13




