
'''simple docstring'''





import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase


@require_tokenizers
class 			UpperCamelCase__	(     unittest.TestCase):
 UpperCAmelCase__     :     Optional[Any]    					=       EsmTokenizer






 def 						lowercase_    (      self	:List[Any]      )					->      Dict:







       '''simple docstring'''



       super().setUp()
       __A													=      tempfile.mkdtemp()
       # fmt: off
       __A													=      ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
       # fmt: on
       __A													=      os.path.join(self.tmpdirname						,					VOCAB_FILES_NAMES['vocab_file']      )
       with open(self.vocab_file						,					'w'						,					encoding='utf-8'      ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )






 def 						lowercase_    (      self	:Optional[Any]						,					**_A	:Union[str, Any]      )					->      List[PreTrainedTokenizerBase]:







       '''simple docstring'''



       return [self.get_tokenizer(**_A      )]






 def 						lowercase_    (      self	:Dict						,					**_A	:Union[str, Any]      )					->      PreTrainedTokenizer:







       '''simple docstring'''



       return self.tokenizer_class.from_pretrained(self.tmpdirname						,					**_A      )






 def 						lowercase_    (      self	:int      )					->      int:







       '''simple docstring'''



       __A													=      self.tokenizer_class(self.vocab_file      )

       __A													=      tokenizer.tokenize('LAGVS'      )
       self.assertListEqual(_A						,					['L', 'A', 'G', 'V', 'S']      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )						,					[4, 5, 6, 7, 8]      )






 def 						lowercase_    (      self	:Optional[int]      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      self.tokenizer_class(self.vocab_file      )

       __A													=      'LAGVS'
       self.assertListEqual(tokenizer.encode(_A      )						,					[0, 4, 5, 6, 7, 8, 2]      )






 def 						lowercase_    (      self	:Any      )					->      Dict:







       '''simple docstring'''



       __A													=      self.tokenizer_class(self.vocab_file      )

       __A													=      ['LAGVS', 'WCB']
       __A													=      tokenizer(_A						,					padding=_A      )['input_ids']

       self.assertListEqual(_A						,					[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]      )






 def 						lowercase_    (      self	:List[str]      )					->      Any:







       '''simple docstring'''



       __A													=      self.tokenizer_class(self.vocab_file      )

       __A													=      ['LAGVS', 'WCB']
       __A													=      tokenizer(_A						,					padding=_A      )['input_ids']

       self.assertListEqual(_A						,					[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]      )






 def 						lowercase_    (      self	:Any      )					->      str:







       '''simple docstring'''



       __A													=      self.get_tokenizers(fast=_A      )
       for tokenizer in tokenizers:
             with self.subTest(F'{tokenizer.__class__.__name__}'      ):
                   __A													=      '<unk>'
                   __A													=      '<mask>'

                   __A													=      tokenizer.tokenize(_A      )
                   __A													=      tokenizer.tokenize(_A      )

                   self.assertEqual(len(_A      )						,					1      )
                   self.assertEqual(len(_A      )						,					1      )
                   self.assertEqual(token_a[0]						,					_A      )
                   self.assertEqual(token_a[0]						,					_A      )














'''simple docstring'''

import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp






def 						_UpperCAmelCase     (	)      ->      Optional[Any]:





						"""simple docstring"""
						lowercase_   : List[str]   	=      ArgumentParser(
						    description=(
						        'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
						    )				)

						# Optional arguments for the launch helper
						parser.add_argument('--num_cores'    ,      type=a    ,      default=1    ,      help='Number of TPU cores to use (1 or 8).'				)

						# positional
						parser.add_argument(
						    'training_script'    ,      type=a    ,      help=(
						        'The full path to the single TPU training '
						        'program/script to be launched in parallel, '
						        'followed by all the arguments for the '
						        'training script'
						    )    ,      )

						# rest from the training program
						parser.add_argument('training_script_args'    ,      nargs=a				)

						return parser.parse_args()






def 						_UpperCAmelCase     (	)      ->      str:





						"""simple docstring"""
						lowercase_   : Optional[int]   	=      parse_args()

						# Import training_script as a module.
						lowercase_   : str   	=      Path(args.training_script				)
						sys.path.append(str(script_fpath.parent.resolve()				)				)
						lowercase_   : int   	=      script_fpath.stem
						lowercase_   : List[Any]   	=      importlib.import_module(a				)

						# Patch sys.argv
						lowercase_   : List[str]   	=      [args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores				)]

						xmp.spawn(mod._mp_fn    ,      args=()    ,      nprocs=args.num_cores				)


if __name__ == "__main__":
						main()
