
import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_				:							Any       =    logging.get_logger(__name__)

lowerCamelCase_				:							Any       =    {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowerCamelCase_				:							int       =    {
    "vocab_file": {
        "microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json",
    },
    "merges_file": {"microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"},
}

lowerCamelCase_				:							Optional[int]       =    {
    "microsoft/biogpt": 1_024,
}
def        __lowercase(							__snake_case				:		Optional[Any]					)       ->   str:
       __snake_case						       =   set()
       __snake_case						       =   word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              __snake_case						       =   char
       return pairs







class  _lowerCamelCase			(lowerCamelCase					):
     lowercase__				  =			VOCAB_FILES_NAMES
     lowercase__				  =			PRETRAINED_VOCAB_FILES_MAP
     lowercase__				  =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase__				  =			["""input_ids""", """attention_mask"""]

     def __init__(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_="<unk>"							,       SCREAMING_SNAKE_CASE_="<s>"							,       SCREAMING_SNAKE_CASE_="</s>"							,       SCREAMING_SNAKE_CASE_="</s>"							,       SCREAMING_SNAKE_CASE_="<pad>"							,       **SCREAMING_SNAKE_CASE_							,       ):
            super().__init__(
                bos_token=SCREAMING_SNAKE_CASE_							,       eos_token=SCREAMING_SNAKE_CASE_							,       sep_token=SCREAMING_SNAKE_CASE_							,       unk_token=SCREAMING_SNAKE_CASE_							,       pad_token=SCREAMING_SNAKE_CASE_							,       **SCREAMING_SNAKE_CASE_							,       )

            try:
                   import sacremoses
            except ImportError:
                   raise ImportError(
                       'You need to install sacremoses to use BioGptTokenizer. '
                       'See https://pypi.org/project/sacremoses/ for installation.'    )

            __snake_case						       =   'en'
            __snake_case						       =   sacremoses
            # cache of sm.MosesTokenizer instance
            __snake_case						       =   {}
            __snake_case						       =   {}
            with open(SCREAMING_SNAKE_CASE_							,       encoding='utf-8'    ) as vocab_handle:
                   __snake_case						       =   json.load(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   {v: k for k, v in self.encoder.items()}
            with open(SCREAMING_SNAKE_CASE_							,       encoding='utf-8'    ) as merges_handle:
                   __snake_case						       =   merges_handle.read().split('\n'    )[:-1]
            __snake_case						       =   [tuple(merge.split()[:2]    ) for merge in merges]
            __snake_case						       =   dict(zip(SCREAMING_SNAKE_CASE_							,       range(len(SCREAMING_SNAKE_CASE_    )    )    )    )
            __snake_case						       =   {}

     @property
     def 						__lowerCamelCase				(  self    ):
            return len(self.encoder    )

     def 						__lowerCamelCase				(  self    ):
            return dict(self.encoder							,       **self.added_tokens_encoder    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            if lang not in self.cache_moses_tokenizer:
                   __snake_case						       =   self.sm.MosesTokenizer(lang=SCREAMING_SNAKE_CASE_    )
                   __snake_case						       =   moses_tokenizer
            return self.cache_moses_tokenizer[lang].tokenize(
                SCREAMING_SNAKE_CASE_							,       aggressive_dash_splits=SCREAMING_SNAKE_CASE_							,       return_str=SCREAMING_SNAKE_CASE_							,       escape=SCREAMING_SNAKE_CASE_    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            if lang not in self.cache_moses_detokenizer:
                   __snake_case						       =   self.sm.MosesDetokenizer(lang=SCREAMING_SNAKE_CASE_    )
                   __snake_case						       =   moses_detokenizer
            return self.cache_moses_detokenizer[lang].detokenize(SCREAMING_SNAKE_CASE_    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   tuple(token[:-1]    ) + (token[-1] + '</w>',)
            if token in self.cache:
                   return self.cache[token]
            __snake_case						       =   get_pairs(SCREAMING_SNAKE_CASE_    )

            if not pairs:
                   return token + "</w>"

            while True:
                   __snake_case						       =   min(SCREAMING_SNAKE_CASE_							,       key=lambda SCREAMING_SNAKE_CASE_    : self.bpe_ranks.get(SCREAMING_SNAKE_CASE_							,       float('inf'    )    )    )
                   if bigram not in self.bpe_ranks:
                          break
                   __snake_case       ,       __snake_case						       =   bigram
                   __snake_case						       =   []
                   __snake_case						       =   0
                   while i < len(SCREAMING_SNAKE_CASE_    ):
                          try:
                                 __snake_case						       =   word.index(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
                          except ValueError:
                                 new_word.extend(word[i:]    )
                                 break
                          else:
                                 new_word.extend(word[i:j]    )
                                 __snake_case						       =   j

                          if word[i] == first and i < len(SCREAMING_SNAKE_CASE_    ) - 1 and word[i + 1] == second:
                                 new_word.append(first + second    )
                                 i += 2
                          else:
                                 new_word.append(word[i]    )
                                 i += 1
                   __snake_case						       =   tuple(SCREAMING_SNAKE_CASE_    )
                   __snake_case						       =   new_word
                   if len(SCREAMING_SNAKE_CASE_    ) == 1:
                          break
                   else:
                          __snake_case						       =   get_pairs(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   ' '.join(SCREAMING_SNAKE_CASE_    )
            if word == "\n  </w>":
                   __snake_case						       =   '\n</w>'
            __snake_case						       =   word
            return word

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_=False    ):
            if bypass_tokenizer:
                   __snake_case						       =   text.split()
            else:
                   __snake_case						       =   self.moses_tokenize(SCREAMING_SNAKE_CASE_							,       self.lang    )

            __snake_case						       =   []
            for token in text:
                   if token:
                          split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_    ).split(' '    )    )    )

            return split_tokens

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            return self.encoder.get(SCREAMING_SNAKE_CASE_							,       self.encoder.get(self.unk_token    )    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            return self.decoder.get(SCREAMING_SNAKE_CASE_							,       self.unk_token    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   [t.replace(' '							,       ''    ).replace('</w>'							,       ' '    ) for t in tokens]
            __snake_case						       =   ''.join(SCREAMING_SNAKE_CASE_    ).split()
            # detokenize
            __snake_case						       =   self.moses_detokenize(SCREAMING_SNAKE_CASE_							,       self.lang    )
            return text

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_ = None    ):
            if token_ids_a is None:
                   return [self.sep_token_id] + token_ids_a
            __snake_case						       =   [self.sep_token_id]
            return sep + token_ids_a + sep + token_ids_a

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_ = None							,       SCREAMING_SNAKE_CASE_ = False    ):
            if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=SCREAMING_SNAKE_CASE_							,       token_ids_a=SCREAMING_SNAKE_CASE_							,       already_has_special_tokens=SCREAMING_SNAKE_CASE_    )
            # no bos used in fairseq
            if token_ids_a is not None:
                   return [1] + ([0] * len(SCREAMING_SNAKE_CASE_    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_    ))
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE_    ))

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_ = None    ):
            __snake_case						       =   [self.sep_token_id]

            # no bos used in fairseq
            if token_ids_a is None:
                   return len(token_ids_a + sep    ) * [0]
            return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_ = None    ):
            if not os.path.isdir(SCREAMING_SNAKE_CASE_    ):
                   logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
                   return
            __snake_case						       =   os.path.join(
                SCREAMING_SNAKE_CASE_							,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
            __snake_case						       =   os.path.join(
                SCREAMING_SNAKE_CASE_							,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

            with open(SCREAMING_SNAKE_CASE_							,       'w'							,       encoding='utf-8'    ) as f:
                   f.write(json.dumps(self.encoder							,       indent=2							,       sort_keys=SCREAMING_SNAKE_CASE_							,       ensure_ascii=SCREAMING_SNAKE_CASE_    ) + '\n'    )

            __snake_case						       =   0
            with open(SCREAMING_SNAKE_CASE_							,       'w'							,       encoding='utf-8'    ) as writer:
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,       key=lambda SCREAMING_SNAKE_CASE_    : kv[1]    ):
                          if index != token_index:
                                 logger.warning(
                                     f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                     ' Please check that the tokenizer is not corrupted!'    )
                                 __snake_case						       =   token_index
                          writer.write(' '.join(SCREAMING_SNAKE_CASE_    ) + '\n'    )
                          index += 1

            return vocab_file, merge_file

     def __getstate__(  self    ):
            __snake_case						       =   self.__dict__.copy()
            __snake_case						       =   None
            return state

     def __setstate__(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   d

            try:
                   import sacremoses
            except ImportError:
                   raise ImportError(
                       'You need to install sacremoses to use XLMTokenizer. '
                       'See https://pypi.org/project/sacremoses/ for installation.'    )

            __snake_case						       =   sacremoses


import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


lowerCamelCase_				:							Optional[Any]       =    logging.get_logger(__name__)

if is_tf_available():
 import tensorflow as tf







@dataclass
class  _lowerCamelCase			(lowerCamelCase					):
     lowercase__				  =			"""tf"""
     lowercase__				  =			field(
         default=lowerCamelCase							,					metadata={"""help""": """Name of TPU"""}							,					)

     lowercase__				  =			field(
         default=lowerCamelCase							,					metadata={"""help""": """Zone of TPU"""}							,					)

     lowercase__				  =			field(
         default=lowerCamelCase							,					metadata={"""help""": """Name of Cloud TPU-enabled project"""}							,					)

     lowercase__				  =			field(
         default=1.0							,					metadata={"""help""": """Power for the Polynomial decay LR scheduler."""}							,					)

     lowercase__				  =			field(default=lowerCamelCase							,					metadata={"""help""": """Whether to activate the XLA compilation or not"""}					)

     @cached_property
     def 						__lowerCamelCase				(  self    ):
            requires_backends(self							,       ['tf']    )
            logger.info('Tensorflow: setting up strategy'    )

            __snake_case						       =   tf.config.list_physical_devices('GPU'    )

            # Set to float16 at first
            if self.fpaa:
                   tf.keras.mixed_precision.set_global_policy('mixed_float16'    )

            if self.no_cuda:
                   __snake_case						       =   tf.distribute.OneDeviceStrategy(device='/cpu:0'    )
            else:
                   try:
                          if self.tpu_name:
                                 __snake_case						       =   tf.distribute.cluster_resolver.TPUClusterResolver(
                                     self.tpu_name							,       zone=self.tpu_zone							,       project=self.gcp_project    )
                          else:
                                 __snake_case						       =   tf.distribute.cluster_resolver.TPUClusterResolver()
                   except ValueError:
                          if self.tpu_name:
                                 raise RuntimeError(f'''Couldn\'t connect to TPU {self.tpu_name}!'''    )
                          else:
                                 __snake_case						       =   None

                   if tpu:
                          # Set to bfloat16 in case of TPU
                          if self.fpaa:
                                 tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'    )

                          tf.config.experimental_connect_to_cluster(SCREAMING_SNAKE_CASE_    )
                          tf.tpu.experimental.initialize_tpu_system(SCREAMING_SNAKE_CASE_    )

                          __snake_case						       =   tf.distribute.TPUStrategy(SCREAMING_SNAKE_CASE_    )

                   elif len(SCREAMING_SNAKE_CASE_    ) == 0:
                          __snake_case						       =   tf.distribute.OneDeviceStrategy(device='/cpu:0'    )
                   elif len(SCREAMING_SNAKE_CASE_    ) == 1:
                          __snake_case						       =   tf.distribute.OneDeviceStrategy(device='/gpu:0'    )
                   elif len(SCREAMING_SNAKE_CASE_    ) > 1:
                          # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
                          __snake_case						       =   tf.distribute.MirroredStrategy()
                   else:
                          raise ValueError('Cannot find the proper strategy, please check your environment properties.'    )

            return strategy

     @property
     def 						__lowerCamelCase				(  self    ):
            requires_backends(self							,       ['tf']    )
            return self._setup_strategy

     @property
     def 						__lowerCamelCase				(  self    ):
            requires_backends(self							,       ['tf']    )
            return self._setup_strategy.num_replicas_in_sync

     @property
     def 						__lowerCamelCase				(  self    ):
            return False  # TF Logging is handled by Keras not the Trainer

     @property
     def 						__lowerCamelCase				(  self    ):
            if self.per_gpu_train_batch_size:
                   logger.warning(
                       'Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '
                       'version. Using `--per_device_train_batch_size` is preferred.'    )
            __snake_case						       =   self.per_gpu_train_batch_size or self.per_device_train_batch_size
            return per_device_batch_size * self.n_replicas

     @property
     def 						__lowerCamelCase				(  self    ):
            if self.per_gpu_eval_batch_size:
                   logger.warning(
                       'Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '
                       'version. Using `--per_device_eval_batch_size` is preferred.'    )
            __snake_case						       =   self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
            return per_device_batch_size * self.n_replicas

     @property
     def 						__lowerCamelCase				(  self    ):
            requires_backends(self							,       ['tf']    )
            warnings.warn(
                'The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.'							,       SCREAMING_SNAKE_CASE_							,       )
            return self._setup_strategy.num_replicas_in_sync
