


import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


__UpperCamelCase							:							Union[str, Any]    							=					logging.get_logger(__name__)

if is_tf_available():
       import tensorflow as tf

@dataclass
class    __magic_name__						(  _SCREAMING_SNAKE_CASE):
     A:		List[str]								=   "tf"
     A:		Optional[str]								=   field(
         default=_SCREAMING_SNAKE_CASE       ,					metadata={"help": "Name of TPU"}       ,					)

     A:		Optional[str]								=   field(
         default=_SCREAMING_SNAKE_CASE       ,					metadata={"help": "Zone of TPU"}       ,					)

     A:		Optional[str]								=   field(
         default=_SCREAMING_SNAKE_CASE       ,					metadata={"help": "Name of Cloud TPU-enabled project"}       ,					)

     A:		float								=   field(
         default=1.0       ,					metadata={"help": "Power for the Polynomial decay LR scheduler."}       ,					)

     A:		bool								=   field(default=_SCREAMING_SNAKE_CASE       ,					metadata={"help": "Whether to activate the XLA compilation or not"})


     @cached_property
     def   UpperCAmelCase__						(   self				:  int      )	->   Optional[int]:





      '''simple docstring'''






      requires_backends(self				,						['''tf''']      )
      logger.info('''Tensorflow: setting up strategy'''      )

      UpperCamelCase__       :     str 							= tf.config.list_physical_devices('''GPU'''      )

      # Set to float16 at first
      if self.fpaa:
       tf.keras.mixed_precision.set_global_policy('''mixed_float16'''      )

      if self.no_cuda:
       UpperCamelCase__       :     Any 							= tf.distribute.OneDeviceStrategy(device='''/cpu:0'''      )
      else:
       try:
        if self.tpu_name:
         UpperCamelCase__       :     Dict 							= tf.distribute.cluster_resolver.TPUClusterResolver(
             self.tpu_name				,						zone=self.tpu_zone				,						project=self.gcp_project      )
        else:
         UpperCamelCase__       :     Optional[int] 							= tf.distribute.cluster_resolver.TPUClusterResolver()
       except ValueError:
        if self.tpu_name:
         raise RuntimeError(F"Couldn't connect to TPU {self.tpu_name}!"      )
        else:
         UpperCamelCase__       :     Optional[Any] 							= None

       if tpu:
        # Set to bfloat16 in case of TPU
        if self.fpaa:
         tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''      )

        tf.config.experimental_connect_to_cluster(lowerCamelCase__      )
        tf.tpu.experimental.initialize_tpu_system(lowerCamelCase__      )

        UpperCamelCase__       :     Optional[int] 							= tf.distribute.TPUStrategy(lowerCamelCase__      )

       elif len(lowerCamelCase__      ) == 0:
        UpperCamelCase__       :     List[str] 							= tf.distribute.OneDeviceStrategy(device='''/cpu:0'''      )
       elif len(lowerCamelCase__      ) == 1:
        UpperCamelCase__       :     int 							= tf.distribute.OneDeviceStrategy(device='''/gpu:0'''      )
       elif len(lowerCamelCase__      ) > 1:
        # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
        UpperCamelCase__       :     Optional[int] 							= tf.distribute.MirroredStrategy()
       else:
        raise ValueError('''Cannot find the proper strategy, please check your environment properties.'''      )

      return strategy


     @property
     def   UpperCAmelCase__						(   self				:  Any      )	->   str:





      '''simple docstring'''






      requires_backends(self				,						['''tf''']      )
      return self._setup_strategy


     @property
     def   UpperCAmelCase__						(   self				:  Union[str, Any]      )	->   List[str]:





      '''simple docstring'''






      requires_backends(self				,						['''tf''']      )
      return self._setup_strategy.num_replicas_in_sync


     @property
     def   UpperCAmelCase__						(   self				:  Tuple      )	->   int:





      '''simple docstring'''






      return False  # TF Logging is handled by Keras not the Trainer


     @property
     def   UpperCAmelCase__						(   self				:  List[Any]      )	->   Dict:





      '''simple docstring'''






      if self.per_gpu_train_batch_size:
       logger.warning(
           '''Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '''
           '''version. Using `--per_device_train_batch_size` is preferred.'''      )
      UpperCamelCase__       :     Dict 							= self.per_gpu_train_batch_size or self.per_device_train_batch_size
      return per_device_batch_size * self.n_replicas


     @property
     def   UpperCAmelCase__						(   self				:  str      )	->   Dict:





      '''simple docstring'''






      if self.per_gpu_eval_batch_size:
       logger.warning(
           '''Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '''
           '''version. Using `--per_device_eval_batch_size` is preferred.'''      )
      UpperCamelCase__       :     str 							= self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
      return per_device_batch_size * self.n_replicas


     @property
     def   UpperCAmelCase__						(   self				:  List[Any]      )	->   Optional[int]:





      '''simple docstring'''






      requires_backends(self				,						['''tf''']      )
      warnings.warn(
          '''The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.'''				,						lowerCamelCase__				,						)
      return self._setup_strategy.num_replicas_in_sync








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
 a_ 		=  None


a_ 		=  logging.get_logger(__name__)

a_ 		=  {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

a_ 		=  {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json"
        ),
    },
}

a_ 		=  {
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}

class 			_lowerCamelCase    (      _SCREAMING_SNAKE_CASE ):

      """simple docstring"""

      lowerCAmelCase__      :			List[Any]     =       VOCAB_FILES_NAMES
      lowerCAmelCase__      :			List[Any]     =       PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__      :			Optional[Any]     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__      :			Optional[int]     =       ["input_ids", "attention_mask"]
      lowerCAmelCase__      :			Union[str, Any]     =       XLMRobertaTokenizer







      def __init__(  self       :			str			,							snake_case       :			Union[str, Any]=None			,							snake_case       :			Any=None			,							snake_case       :			Union[str, Any]="<s>"			,							snake_case       :			Tuple="</s>"			,							snake_case       :			List[Any]="</s>"			,							snake_case       :			Any="<s>"			,							snake_case       :			str="<unk>"			,							snake_case       :			int="<pad>"			,							snake_case       :			Any="<mask>"			,							**snake_case       :			str			,							):
          # Mask token behave like a normal word, i.e. include the space before it
          __UpperCamelCase			=    AddedToken(snake_case			,							lstrip=snake_case			,							rstrip=snake_case					) if isinstance(snake_case			,							snake_case					) else mask_token

          super().__init__(
              snake_case			,							tokenizer_file=snake_case			,							bos_token=snake_case			,							eos_token=snake_case			,							sep_token=snake_case			,							cls_token=snake_case			,							unk_token=snake_case			,							pad_token=snake_case			,							mask_token=snake_case			,							**snake_case			,							)

          __UpperCamelCase			=    vocab_file
          __UpperCamelCase			=    False if not self.vocab_file else True







      def    snake_case						(  self       :			List[Any]			,							snake_case       :			List[int]			,							snake_case       :			Optional[List[int]] = None					):

          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __UpperCamelCase			=    [self.cls_token_id]
          __UpperCamelCase			=    [self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep







      def    snake_case						(  self       :			Any			,							snake_case       :			List[int]			,							snake_case       :			Optional[List[int]] = None					):
          __UpperCamelCase			=    [self.sep_token_id]
          __UpperCamelCase			=    [self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







      def    snake_case						(  self       :			Dict			,							snake_case       :			str			,							snake_case       :			Optional[str] = None					):
          if not self.can_save_slow_tokenizer:
              raise ValueError(
                  '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                  '''tokenizer.'''					)

          if not os.path.isdir(snake_case					):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory."					)
              return
          __UpperCamelCase			=    os.path.join(
              snake_case			,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(snake_case					):
              copyfile(self.vocab_file			,							snake_case					)

          return (out_vocab_file,)






