
import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase    			=					logging.get_logger(__name__)

_UpperCAmelCase    			=					{
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

_UpperCAmelCase    			=					{
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

_UpperCAmelCase    			=					{"stas/tiny-wmt19-en-de": 1024}
_UpperCAmelCase    			=					{
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						set()
  _lowerCamelCase      =						word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)  )
    _lowerCamelCase      =						char
  return pairs


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						text.replace('''，'''			,   ''','''  )
  _lowerCamelCase      =						re.sub(R'''。\s*'''			,   '''. '''			,   _a  )
  _lowerCamelCase      =						text.replace('''、'''			,   ''','''  )
  _lowerCamelCase      =						text.replace('''”'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''“'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''∶'''			,   ''':'''  )
  _lowerCamelCase      =						text.replace('''：'''			,   ''':'''  )
  _lowerCamelCase      =						text.replace('''？'''			,   '''?'''  )
  _lowerCamelCase      =						text.replace('''《'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''》'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''）'''			,   ''')'''  )
  _lowerCamelCase      =						text.replace('''！'''			,   '''!'''  )
  _lowerCamelCase      =						text.replace('''（'''			,   '''('''  )
  _lowerCamelCase      =						text.replace('''；'''			,   ''';'''  )
  _lowerCamelCase      =						text.replace('''１'''			,   '''1'''  )
  _lowerCamelCase      =						text.replace('''」'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''「'''			,   '''"'''  )
  _lowerCamelCase      =						text.replace('''０'''			,   '''0'''  )
  _lowerCamelCase      =						text.replace('''３'''			,   '''3'''  )
  _lowerCamelCase      =						text.replace('''２'''			,   '''2'''  )
  _lowerCamelCase      =						text.replace('''５'''			,   '''5'''  )
  _lowerCamelCase      =						text.replace('''６'''			,   '''6'''  )
  _lowerCamelCase      =						text.replace('''９'''			,   '''9'''  )
  _lowerCamelCase      =						text.replace('''７'''			,   '''7'''  )
  _lowerCamelCase      =						text.replace('''８'''			,   '''8'''  )
  _lowerCamelCase      =						text.replace('''４'''			,   '''4'''  )
  _lowerCamelCase      =						re.sub(R'''．\s*'''			,   '''. '''			,   _a  )
  _lowerCamelCase      =						text.replace('''～'''			,   '''~'''  )
  _lowerCamelCase      =						text.replace('''’'''			,   '''\''''  )
  _lowerCamelCase      =						text.replace('''…'''			,   '''...'''  )
  _lowerCamelCase      =						text.replace('''━'''			,   '''-'''  )
  _lowerCamelCase      =						text.replace('''〈'''			,   '''<'''  )
  _lowerCamelCase      =						text.replace('''〉'''			,   '''>'''  )
  _lowerCamelCase      =						text.replace('''【'''			,   '''['''  )
  _lowerCamelCase      =						text.replace('''】'''			,   ''']'''  )
  _lowerCamelCase      =						text.replace('''％'''			,   '''%'''  )
  return text


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						[]
  for char in text:
    _lowerCamelCase      =						unicodedata.category(_a  )
    if cat.startswith('''C'''  ):
      continue
    output.append(_a  )
  return "".join(_a  )






class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  _UpperCamelCase						 = VOCAB_FILES_NAMES
  _UpperCamelCase						 = PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase						 = PRETRAINED_INIT_CONFIGURATION
  _UpperCamelCase						 = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCamelCase						 = ["input_ids", "attention_mask"]


  def __init__(					self	,		a__=None	,		a__=None	,		a__=None	,		a__=None	,		a__=False	,		a__="<unk>"	,		a__="<s>"	,		a__="</s>"	,		a__="<pad>"	,		**a__	,		):
    super().__init__(
        langs=a__	,		src_vocab_file=a__	,		tgt_vocab_file=a__	,		merges_file=a__	,		do_lower_case=a__	,		unk_token=a__	,		bos_token=a__	,		sep_token=a__	,		pad_token=a__	,		**a__	,		)

    try:
      import sacremoses
    except ImportError:
      raise ImportError(
          '''You need to install sacremoses to use XLMTokenizer. '''
          '''See https://pypi.org/project/sacremoses/ for installation.'''		)

    _lowerCamelCase      =						sacremoses

    _lowerCamelCase      =						src_vocab_file
    _lowerCamelCase      =						tgt_vocab_file
    _lowerCamelCase      =						merges_file
    _lowerCamelCase      =						do_lower_case

    # cache of sm.MosesPunctNormalizer instance
    _lowerCamelCase      =						{}
    # cache of sm.MosesTokenizer instance
    _lowerCamelCase      =						{}
    _lowerCamelCase      =						{}

    if langs and len(a__		) == 2:
      _lowerCamelCase    ,    _lowerCamelCase      =						langs
    else:
      raise ValueError(
          f'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
          '''Usually that means that tokenizer can\'t find a mapping for the given model path '''
          '''in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'''		)

    with open(a__	,		encoding='''utf-8'''		) as src_vocab_handle:
      _lowerCamelCase      =						json.load(a__		)
    with open(a__	,		encoding='''utf-8'''		) as tgt_vocab_handle:
      _lowerCamelCase      =						json.load(a__		)
      _lowerCamelCase      =						{v: k for k, v in tgt_vocab.items()}
    with open(a__	,		encoding='''utf-8'''		) as merges_handle:
      _lowerCamelCase      =						merges_handle.read().split('''\n'''		)[:-1]
    _lowerCamelCase      =						[tuple(merge.split()[:2]		) for merge in merges]
    _lowerCamelCase      =						dict(zip(a__	,		range(len(a__		)		)		)		)
    _lowerCamelCase      =						{}


  def     _UpperCAmelCase  (					self		):
    return self.get_src_vocab()


  @property
  def     _UpperCAmelCase  (					self		):
    return self.src_vocab_size


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    if lang not in self.cache_moses_punct_normalizer:
      _lowerCamelCase      =						self.sm.MosesPunctNormalizer(lang=a__		)
      _lowerCamelCase      =						punct_normalizer
    return self.cache_moses_punct_normalizer[lang].normalize(a__		)


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    if lang not in self.cache_moses_tokenizer:
      _lowerCamelCase      =						self.sm.MosesTokenizer(lang=a__		)
      _lowerCamelCase      =						moses_tokenizer
    return self.cache_moses_tokenizer[lang].tokenize(
        a__	,		aggressive_dash_splits=a__	,		return_str=a__	,		escape=a__		)


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    if lang not in self.cache_moses_detokenizer:
      _lowerCamelCase      =						self.sm.MosesDetokenizer(lang=a__		)
      _lowerCamelCase      =						moses_detokenizer
    return self.cache_moses_detokenizer[lang].detokenize(a__		)


  def     _UpperCAmelCase  (					self	,		a__	,		a__		):
    _lowerCamelCase      =						replace_unicode_punct(a__		)
    _lowerCamelCase      =						self.moses_punct_norm(a__	,		a__		)
    _lowerCamelCase      =						remove_non_printing_char(a__		)
    return text


  @property
  def     _UpperCAmelCase  (					self		):
    return len(self.encoder		)


  @property
  def     _UpperCAmelCase  (					self		):
    return len(self.decoder		)


  def     _UpperCAmelCase  (					self		):
    return dict(self.encoder	,		**self.added_tokens_encoder		)


  def     _UpperCAmelCase  (					self		):
    return dict(self.decoder	,		**self.added_tokens_decoder		)


  def     _UpperCAmelCase  (					self	,		a__		):
    _lowerCamelCase      =						tuple(token[:-1]		) + (token[-1] + '''</w>''',)
    if token in self.cache:
      return self.cache[token]
    _lowerCamelCase      =						get_pairs(a__		)

    if not pairs:
      return token + "</w>"

    while True:
      _lowerCamelCase      =						min(a__	,		key=lambda a__		: self.bpe_ranks.get(a__	,		float('''inf'''		)		)		)
      if bigram not in self.bpe_ranks:
        break
      _lowerCamelCase    ,    _lowerCamelCase      =						bigram
      _lowerCamelCase      =						[]
      _lowerCamelCase      =						0
      while i < len(a__		):
        try:
          _lowerCamelCase      =						word.index(a__	,		a__		)
        except ValueError:
          new_word.extend(word[i:]		)
          break
        else:
          new_word.extend(word[i:j]		)
          _lowerCamelCase      =						j

        if word[i] == first and i < len(a__		) - 1 and word[i + 1] == second:
          new_word.append(first + second		)
          i += 2
        else:
          new_word.append(word[i]		)
          i += 1
      _lowerCamelCase      =						tuple(a__		)
      _lowerCamelCase      =						new_word
      if len(a__		) == 1:
        break
      else:
        _lowerCamelCase      =						get_pairs(a__		)
    _lowerCamelCase      =						''' '''.join(a__		)
    if word == "\n  </w>":
      _lowerCamelCase      =						'''\n</w>'''
    _lowerCamelCase      =						word
    return word


  def     _UpperCAmelCase  (					self	,		a__	,		a__="en"	,		a__=False		):
    _lowerCamelCase      =						self.src_lang

    if self.do_lower_case:
      _lowerCamelCase      =						text.lower()

    if bypass_tokenizer:
      _lowerCamelCase      =						text.split()
    else:
      _lowerCamelCase      =						self.moses_pipeline(a__	,		lang=a__		)
      _lowerCamelCase      =						self.moses_tokenize(a__	,		lang=a__		)

    _lowerCamelCase      =						[]
    for token in text:
      if token:
        split_tokens.extend(list(self.bpe(a__		).split(''' '''		)		)		)

    return split_tokens


  def     _UpperCAmelCase  (					self	,		a__		):
    return self.encoder.get(a__	,		self.encoder.get(self.unk_token		)		)


  def     _UpperCAmelCase  (					self	,		a__		):
    return self.decoder.get(a__	,		self.unk_token		)


  def     _UpperCAmelCase  (					self	,		a__		):
    _lowerCamelCase      =						[t.replace(''' '''	,		''''''		).replace('''</w>'''	,		''' '''		) for t in tokens]
    _lowerCamelCase      =						''''''.join(a__		).split()
    # detokenize
    _lowerCamelCase      =						self.moses_detokenize(a__	,		self.tgt_lang		)
    return text


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    _lowerCamelCase      =						[self.sep_token_id]

    # no bos used in fairseq
    if token_ids_a is None:
      return token_ids_a + sep
    return token_ids_a + sep + token_ids_a + sep


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None	,		a__ = False		):

    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=a__	,		token_ids_a=a__	,		already_has_special_tokens=a__		)
    # no bos used in fairseq
    if token_ids_a is not None:
      return ([0] * len(a__		)) + [1] + ([0] * len(a__		)) + [1]
    return ([0] * len(a__		)) + [1]


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    _lowerCamelCase      =						[self.sep_token_id]

    # no bos used in fairseq
    if token_ids_a is None:
      return len(token_ids_a + sep		) * [0]
    return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    if not os.path.isdir(a__		):
      logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
      return

    _lowerCamelCase      =						os.path.join(
        a__	,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''src_vocab_file''']		)
    _lowerCamelCase      =						os.path.join(
        a__	,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''tgt_vocab_file''']		)
    _lowerCamelCase      =						os.path.join(
        a__	,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

    with open(a__	,		'''w'''	,		encoding='''utf-8'''		) as f:
      f.write(json.dumps(self.encoder	,		indent=2	,		sort_keys=a__	,		ensure_ascii=a__		) + '''\n'''		)

    with open(a__	,		'''w'''	,		encoding='''utf-8'''		) as f:
      _lowerCamelCase      =						{v: k for k, v in self.decoder.items()}
      f.write(json.dumps(a__	,		indent=2	,		sort_keys=a__	,		ensure_ascii=a__		) + '''\n'''		)

    _lowerCamelCase      =						0
    with open(a__	,		'''w'''	,		encoding='''utf-8'''		) as writer:
      for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,		key=lambda a__		: kv[1]		):
        if index != token_index:
          logger.warning(
              f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
              ''' Please check that the tokenizer is not corrupted!'''		)
          _lowerCamelCase      =						token_index
        writer.write(''' '''.join(a__		) + '''\n'''		)
        index += 1

    return src_vocab_file, tgt_vocab_file, merges_file


  def __getstate__(					self		):
    _lowerCamelCase      =						self.__dict__.copy()
    _lowerCamelCase      =						None
    return state


  def __setstate__(					self	,		a__		):
    _lowerCamelCase      =						d

    try:
      import sacremoses
    except ImportError:
      raise ImportError(
          '''You need to install sacremoses to use XLMTokenizer. '''
          '''See https://pypi.org/project/sacremoses/ for installation.'''		)

    _lowerCamelCase      =						sacremoses






import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
  import torch_xla.core.xla_model as xm  # noqa: F401

  _UpperCAmelCase    			=					True
except ImportError:
  _UpperCAmelCase    			=					False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_UpperCAmelCase    			=					torch.distributed.is_available()


def 		_lowerCamelCase       (   _a  ):
  """simple docstring"""




  _lowerCamelCase      =						importlib.util.find_spec(_a  ) is not None
  if package_exists:
    try:
      _lowerCamelCase      =						importlib.metadata.metadata(_a  )
      return True
    except importlib.metadata.PackageNotFoundError:
      return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _torch_distributed_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  try:
    pass
  except ImportError:
    print(
        '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
        ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
        ''' Bindings for PyTorch*.'''  )
  return (
      importlib.util.find_spec('''torch_ccl'''  ) is not None
      or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''  ) is not None
  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return importlib.metadata.version('''oneccl_bind_pt'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformer_engine'''  )


@lru_cache
def 		_lowerCamelCase       (   _a=True  ):
  """simple docstring"""




  if torch.cuda.is_available():
    return False
  if _tpu_available and check_device:
    try:
      # Will raise a RuntimeError if no XLA configuration is found
      _lowerCamelCase      =						xm.xla_device()
      return True
    except RuntimeError:
      return False
  return _tpu_available


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''deepspeed'''  )


def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if is_tpu_available():
    return not ignore_tpu
  if torch.cuda.is_available():
    return torch.cuda.is_bfaa_supported()
  if is_npu_available():
    return False
  return True


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.39.0'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  _lowerCamelCase      =						_is_package_available('''bitsandbytes'''  )
  if package_exists:
    _lowerCamelCase      =						version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
    return compare_versions(_a			,   '''>='''			,   '''0.37.2'''  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''bitsandbytes'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''			,   '''False'''  )  ) == 1:
    _lowerCamelCase      =						importlib.util.find_spec('''megatron'''  ) is not None
    if package_exists:
      try:
        _lowerCamelCase      =						parse(importlib.metadata.version('''megatron-lm'''  )  )
        return compare_versions(_a			,   '''>='''			,   '''2.2.0'''  )
      except Exception as e:
        warnings.warn(F'''Parse Megatron version failed. Exception:{e}'''  )
        return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''safetensors'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''transformers'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''datasets'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''aim'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tensorboard'''  ) or _is_package_available('''tensorboardX'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''wandb'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''comet_ml'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''boto3'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  if _is_package_available('''rich'''  ):
    if "ACCELERATE_DISABLE_RICH" in os.environ:
      warnings.warn(
          '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''  )
      return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''			,   _a  )
    return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''			,   _a  )
  return False


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''sagemaker'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''tqdm'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return _is_package_available('''mlflow'''  )


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  return is_torch_version('''>='''			,   '''1.12'''  ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()


def 		_lowerCamelCase       (   ):
  """simple docstring"""




  def get_major_and_minor_from_version(_a  ):
    return str(version.parse(_a  ).major  ) + "." + str(version.parse(_a  ).minor  )

  _lowerCamelCase      =						importlib.metadata.version('''torch'''  )
  if importlib.util.find_spec('''intel_extension_for_pytorch'''  ) is None:
    return False
  _lowerCamelCase      =						'''N/A'''
  try:
    _lowerCamelCase      =						importlib.metadata.version('''intel_extension_for_pytorch'''  )
  except importlib.metadata.PackageNotFoundError:
    return False
  _lowerCamelCase      =						get_major_and_minor_from_version(_torch_version  )
  _lowerCamelCase      =						get_major_and_minor_from_version(_ipex_version  )
  if torch_major_and_minor != ipex_major_and_minor:
    warnings.warn(
        F'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
        F''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''  )
    return False
  return True


@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if importlib.util.find_spec('''torch'''  ) is None or importlib.util.find_spec('''torch_npu'''  ) is None:
    return False

  import torch
  import torch_npu  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no NPU is found
      _lowerCamelCase      =						torch.npu.device_count()
      return torch.npu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''npu'''  ) and torch.npu.is_available()






@lru_cache
def 		_lowerCamelCase       (   _a=False  ):
  """simple docstring"""




  if not parse_flag_from_env('''ACCELERATE_USE_XPU'''			,   default=_a  ):
    return False
  if is_ipex_available():
    import torch

    if is_torch_version('''<='''			,   '''1.12'''  ):
      return False
  else:
    return False

  import intel_extension_for_pytorch  # noqa: F401

  if check_device:
    try:
      # Will raise a RuntimeError if no XPU  is found
      _lowerCamelCase      =						torch.xpu.device_count()
      return torch.xpu.is_available()
    except RuntimeError:
      return False
  return hasattr(_a			,   '''xpu'''  ) and torch.xpu.is_available()




