
import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY




@is_pipeline_test
class     _SCREAMING_SNAKE_CASE     (      unittest.TestCase						):
	lowerCAmelCase__         = MODEL_FOR_MASKED_LM_MAPPING
	lowerCAmelCase__         = TF_MODEL_FOR_MASKED_LM_MAPPING



	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[Any]:
					super().tearDown()
					# clean-up as much as possible GPU memory occupied by PyTorch
					gc.collect()
					if is_torch_available():
									import torch

									torch.cuda.empty_cache()



	@require_tf
	def     SCREAMING_SNAKE_CASE_(    self )				->       Tuple:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="sshleifer/tiny-distilroberta-base"						,  top_k=2						,  framework="tf" )
					lowerCamelCase_						=			unmasker("My name is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {"sequence": "My name is grouped", "score": 2.1e-05, "token": 38015, "token_str": " grouped"},
					        {"sequence": "My name is accuser", "score": 2.1e-05, "token": 25506, "token_str": " accuser"},
					    ]						,  )

					lowerCamelCase_						=			unmasker("The largest city in France is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {
					            "sequence": "The largest city in France is grouped",
					            "score": 2.1e-05,
					            "token": 38015,
					            "token_str": " grouped",
					        },
					        {
					            "sequence": "The largest city in France is accuser",
					            "score": 2.1e-05,
					            "token": 25506,
					            "token_str": " accuser",
					        },
					    ]						,  )

					lowerCamelCase_						=			unmasker("My name is <mask>"						,  targets=[" Patrick", " Clara", " Teven"]						,  top_k=3 )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
					        {"sequence": "My name is Patrick", "score": 2e-05, "token": 3499, "token_str": " Patrick"},
					        {"sequence": "My name is Te", "score": 1.9e-05, "token": 2941, "token_str": " Te"},
					    ]						,  )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       List[Any]:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="sshleifer/tiny-distilroberta-base"						,  top_k=2						,  framework="pt" )

					lowerCamelCase_						=			unmasker("My name is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {"sequence": "My name is Maul", "score": 2.2e-05, "token": 35676, "token_str": " Maul"},
					        {"sequence": "My name isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
					    ]						,  )

					lowerCamelCase_						=			unmasker("The largest city in France is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {
					            "sequence": "The largest city in France is Maul",
					            "score": 2.2e-05,
					            "token": 35676,
					            "token_str": " Maul",
					        },
					        {"sequence": "The largest city in France isELS", "score": 2.2e-05, "token": 16416, "token_str": "ELS"},
					    ]						,  )

					lowerCamelCase_						=			unmasker("My name is <mask>"						,  targets=[" Patrick", " Clara", " Teven"]						,  top_k=3 )
					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        {"sequence": "My name is Patrick", "score": 2.1e-05, "token": 3499, "token_str": " Patrick"},
					        {"sequence": "My name is Te", "score": 2e-05, "token": 2941, "token_str": " Te"},
					        {"sequence": "My name is Clara", "score": 2e-05, "token": 13606, "token_str": " Clara"},
					    ]						,  )

					lowerCamelCase_						=			unmasker("My name is <mask> <mask>"						,  top_k=2 )

					self.assertEqual(
					    nested_simplify(lowercase						,  decimals=6 )						,  [
					        [
					            {
					                "score": 2.2e-05,
					                "token": 35676,
					                "token_str": " Maul",
					                "sequence": "<s>My name is Maul<mask></s>",
					            },
					            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
					        ],
					        [
					            {
					                "score": 2.2e-05,
					                "token": 35676,
					                "token_str": " Maul",
					                "sequence": "<s>My name is<mask> Maul</s>",
					            },
					            {"score": 2.2e-05, "token": 16416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
					        ],
					    ]						,  )



	@require_torch_gpu
	def     SCREAMING_SNAKE_CASE_(    self )				->       Union[str, Any]:
					lowerCamelCase_						=			pipeline("fill-mask"						,  model="hf-internal-testing/tiny-random-distilbert"						,  device=0						,  framework="pt" )

					# convert model to fp16
					pipe.model.half()

					lowerCamelCase_						=			pipe("Paris is the [MASK] of France." )
					# We actually don't care about the result, we just want to make sure
					# it works, meaning the float16 tensor got casted back to float32
					# for postprocessing.
					self.assertIsInstance(lowercase						,  lowercase )



	@slow
	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[int]:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="distilroberta-base"						,  top_k=2						,  framework="pt" )
					self.run_large_test(lowercase )



	@slow
	@require_tf
	def     SCREAMING_SNAKE_CASE_(    self )				->       Optional[Any]:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="distilroberta-base"						,  top_k=2						,  framework="tf" )
					self.run_large_test(lowercase )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase )				->       int:
					lowerCamelCase_						=			unmasker("My name is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase )						,  [
					        {"sequence": "My name is John", "score": 0.0_0_8, "token": 610, "token_str": " John"},
					        {"sequence": "My name is Chris", "score": 0.0_0_7, "token": 1573, "token_str": " Chris"},
					    ]						,  )
					lowerCamelCase_						=			unmasker("The largest city in France is <mask>" )
					self.assertEqual(
					    nested_simplify(lowercase )						,  [
					        {
					            "sequence": "The largest city in France is Paris",
					            "score": 0.2_5_1,
					            "token": 2201,
					            "token_str": " Paris",
					        },
					        {
					            "sequence": "The largest city in France is Lyon",
					            "score": 0.2_1_4,
					            "token": 12790,
					            "token_str": " Lyon",
					        },
					    ]						,  )

					lowerCamelCase_						=			unmasker("My name is <mask>"						,  targets=[" Patrick", " Clara", " Teven"]						,  top_k=3 )
					self.assertEqual(
					    nested_simplify(lowercase )						,  [
					        {"sequence": "My name is Patrick", "score": 0.0_0_5, "token": 3499, "token_str": " Patrick"},
					        {"sequence": "My name is Clara", "score": 0.0_0_0, "token": 13606, "token_str": " Clara"},
					        {"sequence": "My name is Te", "score": 0.0_0_0, "token": 2941, "token_str": " Te"},
					    ]						,  )



	@require_torch
	def     SCREAMING_SNAKE_CASE_(    self )				->       int:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="sshleifer/tiny-distilroberta-base"						,  framework="pt" )
					lowerCamelCase_						=			None
					lowerCamelCase_						=			None
					self.run_pipeline_test(lowercase						,  [] )



	@require_tf
	def     SCREAMING_SNAKE_CASE_(    self )				->       List[Any]:
					lowerCamelCase_						=			pipeline(task="fill-mask"						,  model="sshleifer/tiny-distilroberta-base"						,  framework="tf" )
					lowerCamelCase_						=			None
					lowerCamelCase_						=			None
					self.run_pipeline_test(lowercase						,  [] )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase						,  lowercase )				->       Union[str, Any]:
					if tokenizer is None or tokenizer.mask_token_id is None:
									self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)" )

					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )
					lowerCamelCase_						=			[
					    f'This is another {tokenizer.mask_token} test',
					]
					return fill_masker, examples



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       Dict:
					lowerCamelCase_						=			fill_masker.tokenizer
					lowerCamelCase_						=			fill_masker.model

					lowerCamelCase_						=			fill_masker(
					    f'This is a {tokenizer.mask_token}'						,  )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )

					lowerCamelCase_						=			fill_masker([f'This is a {tokenizer.mask_token}'] )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )

					lowerCamelCase_						=			fill_masker([f'This is a {tokenizer.mask_token}', f'Another {tokenizer.mask_token} great test.'] )
					self.assertEqual(
					    lowercase						,  [
					        [
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        ],
					        [
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        ],
					    ]						,  )

					with self.assertRaises(lowercase ):
									fill_masker([None] )
					# No mask_token is not supported
					with self.assertRaises(lowercase ):
									fill_masker("This is" )

					self.run_test_top_k(lowercase						,  lowercase )
					self.run_test_targets(lowercase						,  lowercase )
					self.run_test_top_k_targets(lowercase						,  lowercase )
					self.fill_mask_with_duplicate_targets_and_top_k(lowercase						,  lowercase )
					self.fill_mask_with_multiple_masks(lowercase						,  lowercase )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       Dict:
					lowerCamelCase_						=			tokenizer.get_vocab()
					lowerCamelCase_						=			sorted(vocab.keys() )[:2]
					# Pipeline argument
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase						,  targets=lowercase )
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}' )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )
					lowerCamelCase_						=			{vocab[el] for el in targets}
					self.assertEqual({el["token"] for el in outputs}						,  lowercase )
					lowerCamelCase_						=			[tokenizer.decode([x] ) for x in target_ids]
					self.assertEqual({el["token_str"] for el in outputs}						,  set(lowercase ) )

					# Call argument
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets=lowercase )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )
					lowerCamelCase_						=			{vocab[el] for el in targets}
					self.assertEqual({el["token"] for el in outputs}						,  lowercase )
					lowerCamelCase_						=			[tokenizer.decode([x] ) for x in target_ids]
					self.assertEqual({el["token_str"] for el in outputs}						,  set(lowercase ) )

					# Score equivalence
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets=lowercase )
					lowerCamelCase_						=			[top_mask["token_str"] for top_mask in outputs]
					lowerCamelCase_						=			[top_mask["score"] for top_mask in outputs]

					# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
					if set(lowercase ) == set(lowercase ):
									lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets=lowercase )
									lowerCamelCase_						=			[top_mask["score"] for top_mask in unmasked_targets]
									self.assertEqual(nested_simplify(lowercase )						,  nested_simplify(lowercase ) )

					# Raises with invalid
					with self.assertRaises(lowercase ):
									lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets=[] )
					# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
					if "" not in tokenizer.get_vocab():
									with self.assertRaises(lowercase ):
													lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets=[""] )
									with self.assertRaises(lowercase ):
													lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  targets="" )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       List[Any]:
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase						,  top_k=2 )
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}' )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )

					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  top_k=2 )
					self.assertEqual(
					    lowercase						,  [
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					    ]						,  )
					self.assertEqual(nested_simplify(lowercase )						,  nested_simplify(lowercase ) )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       List[Any]:
					lowerCamelCase_						=			tokenizer.get_vocab()
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )

					# top_k=2, ntargets=3
					lowerCamelCase_						=			sorted(vocab.keys() )[:3]
					lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  top_k=2						,  targets=lowercase )

					# If we use the most probably targets, and filter differently, we should still
					# have the same results
					lowerCamelCase_						=			[el["token_str"] for el in sorted(lowercase						,  key=lambda lowercase : x["score"]						,  reverse=lowercase )]
					# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
					if set(lowercase ).issubset(lowercase ):
									lowerCamelCase_						=			fill_masker(f'This is a {tokenizer.mask_token}'						,  top_k=3						,  targets=lowercase )
									# They should yield exactly the same result
									self.assertEqual(nested_simplify(lowercase )						,  nested_simplify(lowercase ) )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       List[Any]:
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )
					lowerCamelCase_						=			tokenizer.get_vocab()
					# String duplicates + id duplicates
					lowerCamelCase_						=			sorted(vocab.keys() )[:3]
					lowerCamelCase_						=			[targets[0], targets[1], targets[0], targets[2], targets[1]]
					lowerCamelCase_						=			fill_masker(f'My name is {tokenizer.mask_token}'						,  targets=lowercase						,  top_k=10 )

					# The target list contains duplicates, so we can't output more
					# than them
					self.assertEqual(len(lowercase )						,  3 )



	def     SCREAMING_SNAKE_CASE_(    self						,  lowercase						,  lowercase )				->       Dict:
					lowerCamelCase_						=			FillMaskPipeline(model=lowercase						,  tokenizer=lowercase )

					lowerCamelCase_						=			fill_masker(
					    f'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'						,  top_k=2 )
					self.assertEqual(
					    lowercase						,  [
					        [
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        ],
					        [
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        ],
					        [
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					            {"sequence": ANY(lowercase ), "score": ANY(lowercase ), "token": ANY(lowercase ), "token_str": ANY(lowercase )},
					        ],
					    ]						,  )






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class     UpperCAmelCase__       (					A_   ,  unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		Tuple      =  TransfoXLTokenizer
    UpperCAmelCase__						:		str      =  False
    UpperCAmelCase__						:		Tuple      =  False






    def _a					(			self    )  ->       Union[str, Any]:
         super().setUp()

         __UpperCamelCase    							=[
             '<unk>',
             '[CLS]',
             '[SEP]',
             'want',
             'unwanted',
             'wa',
             'un',
             'running',
             ',',
             'low',
             'l',
         ]
         __UpperCamelCase    							=os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']    )
         with open(self.vocab_file					,   'w'					,   encoding='utf-8'    ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )






    def _a					(			self					,   **A_    )  ->       Optional[int]:
         __UpperCamelCase    							=True
         return TransfoXLTokenizer.from_pretrained(self.tmpdirname					,   **A_    )






    def _a					(			self					,   A_    )  ->       Tuple:
         __UpperCamelCase    							='<unk> UNwanted , running'
         __UpperCamelCase    							='<unk> unwanted, running'
         return input_text, output_text






    def _a					(			self    )  ->       str:
         __UpperCamelCase    							=TransfoXLTokenizer(vocab_file=self.vocab_file					,   lower_case=A_    )

         __UpperCamelCase    							=tokenizer.tokenize('<unk> UNwanted , running'    )
         self.assertListEqual(A_					,   ['<unk>', 'unwanted', ',', 'running']    )

         self.assertListEqual(tokenizer.convert_tokens_to_ids(A_    )					,   [0, 4, 8, 7]    )






    def _a					(			self    )  ->       Any:
         __UpperCamelCase    							=TransfoXLTokenizer(lower_case=A_    )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '    )					,   ['hello', '!', 'how', 'are', 'you', '?']    )






    def _a					(			self    )  ->       Optional[int]:
         __UpperCamelCase    							=TransfoXLTokenizer(lower_case=A_    )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '    )					,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )






    def _a					(			self    )  ->       int:
         __UpperCamelCase    							=TransfoXLTokenizer(lower_case=A_    )
         __UpperCamelCase    							='Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
         __UpperCamelCase    							=[
             'Hello',
             '(',
             'bracket',
             ')',
             'and',
             'side',
             '@-@',
             'scrolled',
             '[',
             'and',
             ']',
             'Henry',
             '\'s',
             '$',
             '5',
             '@,@',
             '000',
             'with',
             '3',
             '@.@',
             '34',
             'm',
             '.',
             'What',
             '\'s',
             'up',
             '!',
             '?',
         ]

         self.assertListEqual(tokenizer.tokenize(A_    )					,   A_    )

         self.assertEqual(tokenizer.convert_tokens_to_string(A_    )					,   A_    )






    def _a					(			self    )  ->       Optional[int]:
         __UpperCamelCase    							=self.get_tokenizer()
         __UpperCamelCase    							=len(A_    )

         tokenizer.add_tokens(['new1', 'new2']    )
         tokenizer.move_added_token('new1'					,   1    )

         # Check that moved token is not copied (duplicate)
         self.assertEqual(len(A_    )					,   original_len + 2    )
         # Check that token is moved to specified id
         self.assertEqual(tokenizer.encode('new1'    )					,   [1]    )
         self.assertEqual(tokenizer.decode([1]    )					,   'new1'    )





