






"""simple docstring"""


import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


_UpperCamelCase							:       Optional[Any]       					=		logging.get_logger(__name__)

if is_tf_available():
   import tensorflow as tf






@dataclass
class 		snake_case    (    UpperCAmelCase			):
    __magic_name__       	=	'''tf'''
    __magic_name__       	=	field(
        default=UpperCAmelCase  ,					metadata={'''help''': '''Name of TPU'''}  ,					)

    __magic_name__       	=	field(
        default=UpperCAmelCase  ,					metadata={'''help''': '''Zone of TPU'''}  ,					)

    __magic_name__       	=	field(
        default=UpperCAmelCase  ,					metadata={'''help''': '''Name of Cloud TPU-enabled project'''}  ,					)

    __magic_name__       	=	field(
        default=1.0  ,					metadata={'''help''': '''Power for the Polynomial decay LR scheduler.'''}  ,					)

    __magic_name__       	=	field(default=UpperCAmelCase  ,					metadata={'''help''': '''Whether to activate the XLA compilation or not'''}			)





    @cached_property
    def 							lowerCamelCase__ (				self      :   int     ):


      '''simple docstring'''




      requires_backends(self							,						['tf']     )
      logger.info('Tensorflow: setting up strategy'     )

      a	:     str							=  tf.config.list_physical_devices('GPU'     )

      # Set to float16 at first
      if self.fpaa:
        tf.keras.mixed_precision.set_global_policy('mixed_float16'     )

      if self.no_cuda:
        a	:     List[Any]							=  tf.distribute.OneDeviceStrategy(device='/cpu:0'     )
      else:
        try:
          if self.tpu_name:
            a	:     Tuple							=  tf.distribute.cluster_resolver.TPUClusterResolver(
                self.tpu_name							,						zone=self.tpu_zone							,						project=self.gcp_project     )
          else:
            a	:     int							=  tf.distribute.cluster_resolver.TPUClusterResolver()
        except ValueError:
          if self.tpu_name:
            raise RuntimeError(F'''Couldn\'t connect to TPU {self.tpu_name}!'''     )
          else:
            a	:     Tuple							=  None

        if tpu:
          # Set to bfloat16 in case of TPU
          if self.fpaa:
            tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'     )

          tf.config.experimental_connect_to_cluster(SCREAMING_SNAKE_CASE_     )
          tf.tpu.experimental.initialize_tpu_system(SCREAMING_SNAKE_CASE_     )

          a	:     Tuple							=  tf.distribute.TPUStrategy(SCREAMING_SNAKE_CASE_     )

        elif len(SCREAMING_SNAKE_CASE_     ) == 0:
          a	:     List[str]							=  tf.distribute.OneDeviceStrategy(device='/cpu:0'     )
        elif len(SCREAMING_SNAKE_CASE_     ) == 1:
          a	:     Optional[int]							=  tf.distribute.OneDeviceStrategy(device='/gpu:0'     )
        elif len(SCREAMING_SNAKE_CASE_     ) > 1:
          # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
          a	:     int							=  tf.distribute.MirroredStrategy()
        else:
          raise ValueError('Cannot find the proper strategy, please check your environment properties.'     )

      return strategy





    @property
    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      requires_backends(self							,						['tf']     )
      return self._setup_strategy





    @property
    def 							lowerCamelCase__ (				self      :   Optional[Any]     ):


      '''simple docstring'''




      requires_backends(self							,						['tf']     )
      return self._setup_strategy.num_replicas_in_sync





    @property
    def 							lowerCamelCase__ (				self      :   Optional[int]     ):


      '''simple docstring'''




      return False  # TF Logging is handled by Keras not the Trainer





    @property
    def 							lowerCamelCase__ (				self      :   Dict     ):


      '''simple docstring'''




      if self.per_gpu_train_batch_size:
        logger.warning(
            'Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '
            'version. Using `--per_device_train_batch_size` is preferred.'     )
      a	:     Union[str, Any]							=  self.per_gpu_train_batch_size or self.per_device_train_batch_size
      return per_device_batch_size * self.n_replicas





    @property
    def 							lowerCamelCase__ (				self      :   Tuple     ):


      '''simple docstring'''




      if self.per_gpu_eval_batch_size:
        logger.warning(
            'Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '
            'version. Using `--per_device_eval_batch_size` is preferred.'     )
      a	:     Tuple							=  self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
      return per_device_batch_size * self.n_replicas





    @property
    def 							lowerCamelCase__ (				self      :   str     ):


      '''simple docstring'''




      requires_backends(self							,						['tf']     )
      warnings.warn(
          'The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.'							,						SCREAMING_SNAKE_CASE_							,						)
      return self._setup_strategy.num_replicas_in_sync






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class  _lowerCamelCase			(lowerCamelCase							,					unittest.TestCase					):
     lowercase__				  =			HerbertTokenizer
     lowercase__				  =			HerbertTokenizerFast
     lowercase__				  =			True

     def 						__lowerCamelCase				(  self    ):
            super().setUp()

            # Use a simpler test file without japanese/chinese characters
            with open(f'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''							,       encoding='utf-8'    ) as f_data:
                   __snake_case						       =   f_data.read().replace('\n\n'							,       '\n'    ).strip()

            __snake_case						       =   [
                '<s>',
                '</s>',
                'l',
                'o',
                'w',
                'e',
                'r',
                's',
                't',
                'i',
                'd',
                'n',
                'w</w>',
                'r</w>',
                't</w>',
                'lo',
                'low',
                'er</w>',
                'low</w>',
                'lowest</w>',
                'newer</w>',
                'wider</w>',
                ',</w>',
                '<unk>',
            ]
            __snake_case						       =   dict(zip(SCREAMING_SNAKE_CASE_							,       range(len(SCREAMING_SNAKE_CASE_    )    )    )    )
            __snake_case						       =   ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

            __snake_case						       =   os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['vocab_file']    )
            __snake_case						       =   os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['merges_file']    )
            with open(self.vocab_file							,       'w'    ) as fp:
                   fp.write(json.dumps(SCREAMING_SNAKE_CASE_    )    )
            with open(self.merges_file							,       'w'    ) as fp:
                   fp.write('\n'.join(SCREAMING_SNAKE_CASE_    )    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   'lower newer'
            __snake_case						       =   'lower newer'
            return input_text, output_text

     def 						__lowerCamelCase				(  self    ):
            __snake_case						       =   self.tokenizer_class(vocab_file=self.vocab_file							,       merges_file=self.merges_file    )

            __snake_case						       =   'lower'
            __snake_case						       =   ['low', 'er</w>']
            __snake_case						       =   tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokens + ['<unk>']
            __snake_case						       =   [16, 17, 23]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_    )							,       SCREAMING_SNAKE_CASE_    )

     def 						__lowerCamelCase				(  self    ):
            if not self.test_rust_tokenizer:
                   return

            __snake_case						       =   self.get_tokenizer()
            __snake_case						       =   self.get_rust_tokenizer()

            __snake_case						       =   'lower,newer'

            __snake_case						       =   tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokenizer.encode(SCREAMING_SNAKE_CASE_							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   self.get_rust_tokenizer()
            __snake_case						       =   tokenizer.encode(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

     @slow
     def 						__lowerCamelCase				(  self    ):
            __snake_case						       =   self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'    )

            __snake_case						       =   tokenizer.encode('konstruowanie sekwencji'							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   tokenizer.encode('konstruowanie wielu sekwencji'							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            assert encoded_sentence == [0] + text + [2]
            assert encoded_pair == [0] + text + [2] + text_a + [2]
