





import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp






def lowerCamelCase__    (							)      ->				Optional[int]:
   __snake_case							       =   ArgumentParser(
       description=(
           '''PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'''
       )       )

   # Optional arguments for the launch helper
   parser.add_argument('''--num_cores'''       ,      type=snake_case_       ,      default=1       ,      help='''Number of TPU cores to use (1 or 8).'''       )

   # positional
   parser.add_argument(
       '''training_script'''       ,      type=snake_case_       ,      help=(
           '''The full path to the single TPU training '''
           '''program/script to be launched in parallel, '''
           '''followed by all the arguments for the '''
           '''training script'''
       )       ,      )

   # rest from the training program
   parser.add_argument('''training_script_args'''       ,      nargs=snake_case_       )

   return parser.parse_args()






def lowerCamelCase__    (							)      ->				List[Any]:
   __snake_case							       =   parse_args()

   # Import training_script as a module.
   __snake_case							       =   Path(args.training_script       )
   sys.path.append(str(script_fpath.parent.resolve()       )       )
   __snake_case							       =   script_fpath.stem
   __snake_case							       =   importlib.import_module(snake_case_       )

   # Patch sys.argv
   __snake_case							       =   [args.training_script] + args.training_script_args + ['''--tpu_num_cores''', str(args.num_cores       )]

   xmp.spawn(mod._mp_fn       ,      args=()       ,      nprocs=args.num_cores       )


if __name__ == "__main__":
    main()








import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin

@require_jieba
class 		SCREAMING_SNAKE_CASE__				(       _UpperCAmelCase						,      unittest.TestCase  ):
      A_						:				List[str]							=    CpmAntTokenizer
      A_						:				Optional[int]							=    False


      def    a (self       :      Optional[int]      ):





         """simple docstring"""



         super().setUp()

         __snake_case							       =   [
             '''<d>''',
             '''</d>''',
             '''<s>''',
             '''</s>''',
             '''</_>''',
             '''<unk>''',
             '''<pad>''',
             '''</n>''',
             '''我''',
             '''是''',
             '''C''',
             '''P''',
             '''M''',
             '''A''',
             '''n''',
             '''t''',
         ]
         __snake_case							       =   os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES['''vocab_file''']      )
         with open(self.vocab_file				,			'''w'''				,			encoding='''utf-8'''      ) as vocab_writer:
            vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )


      @tooslow
      def    a (self       :      Dict      ):





         """simple docstring"""



         __snake_case							       =   CpmAntTokenizer.from_pretrained('''openbmb/cpm-ant-10b'''      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   ['''今天''', '''天气''', '''真''', '''好''', '''！''']
         __snake_case							       =   tokenizer.tokenize(a__      )
         self.assertListEqual(a__				,			a__      )
         __snake_case							       =   '''今天天气真好！'''
         __snake_case							       =   [tokenizer.bos_token] + tokens

         __snake_case							       =   [6, 9802, 1_4962, 2082, 831, 244]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )				,			a__      )

         __snake_case							       =   tokenizer.decode(a__      )
         self.assertEqual(a__				,			a__      )

