
"""simple docstring"""






import argparse
import json

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    VideoMAEConfig,
    VideoMAEForPreTraining,
    VideoMAEForVideoClassification,
    VideoMAEImageProcessor,
)






def 						_snake_case       (	lowercase__		:  Optional[Any]		)  ->				Union[str, Any]:


			'''simple docstring'''





			lowerCAmelCase_    :Any			     =  VideoMAEConfig()

			set_architecture_configs(__A      ,     __A		)

			if "finetuned" not in model_name:
						lowerCAmelCase_    :Union[str, Any]			     =  False

			if "finetuned" in model_name:
						lowerCAmelCase_    :int			     =  """huggingface/label-files"""
						if "kinetics" in model_name:
									lowerCAmelCase_    :List[Any]			     =  4_0_0
									lowerCAmelCase_    :List[str]			     =  """kinetics400-id2label.json"""
						elif "ssv2" in model_name:
									lowerCAmelCase_    :Tuple			     =  1_7_4
									lowerCAmelCase_    :List[str]			     =  """something-something-v2-id2label.json"""
						else:
									raise ValueError("""Model name should either contain \'kinetics\' or \'ssv2\' in case it\'s fine-tuned."""		)
						lowerCAmelCase_    :Tuple			     =  json.load(open(hf_hub_download(__A      ,     __A      ,     repo_type="""dataset"""		)      ,     """r"""		)		)
						lowerCAmelCase_    :int			     =  {int(__A		): v for k, v in idalabel.items()}
						lowerCAmelCase_    :List[str]			     =  idalabel
						lowerCAmelCase_    :Optional[int]			     =  {v: k for k, v in idalabel.items()}

			return config






def 						_snake_case       (	lowercase__		:  str      ,     lowercase__		:  int		)  ->				Optional[Any]:


			'''simple docstring'''





			if "small" in model_name:
						lowerCAmelCase_    :Optional[Any]			     =  3_8_4
						lowerCAmelCase_    :Union[str, Any]			     =  1_5_3_6
						lowerCAmelCase_    :Dict			     =  1_2
						lowerCAmelCase_    :Tuple			     =  1_6
						lowerCAmelCase_    :str			     =  1_2
						lowerCAmelCase_    :Optional[Any]			     =  3
						lowerCAmelCase_    :int			     =  1_9_2
						lowerCAmelCase_    :Any			     =  7_6_8
			elif "large" in model_name:
						lowerCAmelCase_    :Optional[int]			     =  1_0_2_4
						lowerCAmelCase_    :List[Any]			     =  4_0_9_6
						lowerCAmelCase_    :Tuple			     =  2_4
						lowerCAmelCase_    :Dict			     =  1_6
						lowerCAmelCase_    :int			     =  1_2
						lowerCAmelCase_    :Optional[int]			     =  8
						lowerCAmelCase_    :str			     =  5_1_2
						lowerCAmelCase_    :Optional[int]			     =  2_0_4_8
			elif "huge" in model_name:
						lowerCAmelCase_    :Union[str, Any]			     =  1_2_8_0
						lowerCAmelCase_    :int			     =  5_1_2_0
						lowerCAmelCase_    :List[str]			     =  3_2
						lowerCAmelCase_    :List[Any]			     =  1_6
						lowerCAmelCase_    :Optional[Any]			     =  1_2
						lowerCAmelCase_    :Dict			     =  8
						lowerCAmelCase_    :Dict			     =  6_4_0
						lowerCAmelCase_    :List[str]			     =  2_5_6_0
			elif "base" not in model_name:
						raise ValueError("""Model name should include either \"small\", \"base\", \"large\", or \"huge\""""		)






def 						_snake_case       (	lowercase__		:  int		)  ->				Tuple:


			'''simple docstring'''





			if "encoder." in name:
						lowerCAmelCase_    :Tuple			     =  name.replace("""encoder."""      ,     """"""		)
			if "cls_token" in name:
						lowerCAmelCase_    :Optional[Any]			     =  name.replace("""cls_token"""      ,     """videomae.embeddings.cls_token"""		)
			if "decoder_pos_embed" in name:
						lowerCAmelCase_    :Optional[int]			     =  name.replace("""decoder_pos_embed"""      ,     """decoder.decoder_pos_embed"""		)
			if "pos_embed" in name and "decoder" not in name:
						lowerCAmelCase_    :Tuple			     =  name.replace("""pos_embed"""      ,     """videomae.embeddings.position_embeddings"""		)
			if "patch_embed.proj" in name:
						lowerCAmelCase_    :Optional[Any]			     =  name.replace("""patch_embed.proj"""      ,     """videomae.embeddings.patch_embeddings.projection"""		)
			if "patch_embed.norm" in name:
						lowerCAmelCase_    :str			     =  name.replace("""patch_embed.norm"""      ,     """videomae.embeddings.norm"""		)
			if "decoder.blocks" in name:
						lowerCAmelCase_    :List[str]			     =  name.replace("""decoder.blocks"""      ,     """decoder.decoder_layers"""		)
			if "blocks" in name:
						lowerCAmelCase_    :List[Any]			     =  name.replace("""blocks"""      ,     """videomae.encoder.layer"""		)
			if "attn.proj" in name:
						lowerCAmelCase_    :Optional[int]			     =  name.replace("""attn.proj"""      ,     """attention.output.dense"""		)
			if "attn" in name and "bias" not in name:
						lowerCAmelCase_    :Dict			     =  name.replace("""attn"""      ,     """attention.self"""		)
			if "attn" in name:
						lowerCAmelCase_    :Optional[Any]			     =  name.replace("""attn"""      ,     """attention.attention"""		)
			if "norm1" in name:
						lowerCAmelCase_    :Optional[Any]			     =  name.replace("""norm1"""      ,     """layernorm_before"""		)
			if "norm2" in name:
						lowerCAmelCase_    :List[Any]			     =  name.replace("""norm2"""      ,     """layernorm_after"""		)
			if "mlp.fc1" in name:
						lowerCAmelCase_    :Optional[Any]			     =  name.replace("""mlp.fc1"""      ,     """intermediate.dense"""		)
			if "mlp.fc2" in name:
						lowerCAmelCase_    :Tuple			     =  name.replace("""mlp.fc2"""      ,     """output.dense"""		)
			if "decoder_embed" in name:
						lowerCAmelCase_    :str			     =  name.replace("""decoder_embed"""      ,     """decoder.decoder_embed"""		)
			if "decoder_norm" in name:
						lowerCAmelCase_    :str			     =  name.replace("""decoder_norm"""      ,     """decoder.decoder_norm"""		)
			if "decoder_pred" in name:
						lowerCAmelCase_    :List[str]			     =  name.replace("""decoder_pred"""      ,     """decoder.decoder_pred"""		)
			if "norm.weight" in name and "decoder" not in name and "fc" not in name:
						lowerCAmelCase_    :Union[str, Any]			     =  name.replace("""norm.weight"""      ,     """videomae.layernorm.weight"""		)
			if "norm.bias" in name and "decoder" not in name and "fc" not in name:
						lowerCAmelCase_    :Optional[int]			     =  name.replace("""norm.bias"""      ,     """videomae.layernorm.bias"""		)
			if "head" in name and "decoder" not in name:
						lowerCAmelCase_    :Any			     =  name.replace("""head"""      ,     """classifier"""		)

			return name






def 						_snake_case       (	lowercase__		:  Union[str, Any]      ,     lowercase__		:  List[str]		)  ->				Tuple:


			'''simple docstring'''





			for key in orig_state_dict.copy().keys():
						lowerCAmelCase_    :Any			     =  orig_state_dict.pop(__A		)

						if key.startswith("""encoder."""		):
									lowerCAmelCase_    :Dict			     =  key.replace("""encoder."""      ,     """"""		)

						if "qkv" in key:
									lowerCAmelCase_    :List[str]			     =  key.split("""."""		)
									if key.startswith("""decoder.blocks"""		):
												lowerCAmelCase_    :Optional[Any]			     =  config.decoder_hidden_size
												lowerCAmelCase_    :Tuple			     =  int(key_split[2]		)
												lowerCAmelCase_    :List[Any]			     =  """decoder.decoder_layers."""
												if "weight" in key:
															lowerCAmelCase_    :Optional[Any]			     =  val[:dim, :]
															lowerCAmelCase_    :List[str]			     =  val[dim : dim * 2, :]
															lowerCAmelCase_    :List[str]			     =  val[-dim:, :]
									else:
												lowerCAmelCase_    :Dict			     =  config.hidden_size
												lowerCAmelCase_    :Any			     =  int(key_split[1]		)
												lowerCAmelCase_    :Union[str, Any]			     =  """videomae.encoder.layer."""
												if "weight" in key:
															lowerCAmelCase_    :Any			     =  val[:dim, :]
															lowerCAmelCase_    :Union[str, Any]			     =  val[dim : dim * 2, :]
															lowerCAmelCase_    :Optional[int]			     =  val[-dim:, :]
						else:
									lowerCAmelCase_    :Union[str, Any]			     =  val

			return orig_state_dict






def 						_snake_case       (	)  ->				Tuple:


			'''simple docstring'''





			lowerCAmelCase_    :str			     =  hf_hub_download(
			    repo_id="""hf-internal-testing/spaghetti-video"""      ,     filename="""eating_spaghetti.npy"""      ,     repo_type="""dataset"""		)
			lowerCAmelCase_    :Optional[int]			     =  np.load(__A		)
			return list(__A		)






def 						_snake_case       (	lowercase__		:  Optional[Any]      ,     lowercase__		:  Optional[int]      ,     lowercase__		:  Any      ,     lowercase__		:  Tuple		)  ->				List[Any]:


			'''simple docstring'''





			lowerCAmelCase_    :Union[str, Any]			     =  get_videomae_config(__A		)

			if "finetuned" in model_name:
						lowerCAmelCase_    :Dict			     =  VideoMAEForVideoClassification(__A		)
			else:
						lowerCAmelCase_    :str			     =  VideoMAEForPreTraining(__A		)

			# download original checkpoint, hosted on Google Drive
			lowerCAmelCase_    :Optional[int]			     =  """pytorch_model.bin"""
			gdown.cached_download(__A      ,     __A      ,     quiet=__A		)
			lowerCAmelCase_    :Tuple			     =  torch.load(__A      ,     map_location="""cpu"""		)
			if "model" in files:
						lowerCAmelCase_    :Optional[int]			     =  files["""model"""]
			else:
						lowerCAmelCase_    :List[Any]			     =  files["""module"""]
			lowerCAmelCase_    :List[Any]			     =  convert_state_dict(__A      ,     __A		)

			model.load_state_dict(__A		)
			model.eval()

			# verify model on basic input
			lowerCAmelCase_    :List[str]			     =  VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]      ,     image_std=[0.5, 0.5, 0.5]		)
			lowerCAmelCase_    :Tuple			     =  prepare_video()
			lowerCAmelCase_    :List[str]			     =  image_processor(__A      ,     return_tensors="""pt"""		)

			if "finetuned" not in model_name:
						lowerCAmelCase_    :List[Any]			     =  hf_hub_download(repo_id="""hf-internal-testing/bool-masked-pos"""      ,     filename="""bool_masked_pos.pt"""		)
						lowerCAmelCase_    :Optional[Any]			     =  torch.load(__A		)

			lowerCAmelCase_    :int			     =  model(**__A		)
			lowerCAmelCase_    :int			     =  outputs.logits

			lowerCAmelCase_    :List[Any]			     =  [
			    """videomae-small-finetuned-kinetics""",
			    """videomae-small-finetuned-ssv2""",
			    # Kinetics-400 checkpoints (short = pretrained only for 800 epochs instead of 1600)
			    """videomae-base-short""",
			    """videomae-base-short-finetuned-kinetics""",
			    """videomae-base""",
			    """videomae-base-finetuned-kinetics""",
			    """videomae-large""",
			    """videomae-large-finetuned-kinetics""",
			    """videomae-huge-finetuned-kinetics""",
			    # Something-Something-v2 checkpoints (short = pretrained only for 800 epochs instead of 2400)
			    """videomae-base-short-ssv2""",
			    """videomae-base-short-finetuned-ssv2""",
			    """videomae-base-ssv2""",
			    """videomae-base-finetuned-ssv2""",
			]

			# NOTE: logits were tested with image_mean and image_std equal to [0.5, 0.5, 0.5] and [0.5, 0.5, 0.5]
			if model_name == "videomae-small-finetuned-kinetics":
						lowerCAmelCase_    :Optional[int]			     =  torch.Size([1, 4_0_0]		)
						lowerCAmelCase_    :Tuple			     =  torch.tensor([-0.9291, -0.4061, -0.9307]		)
			elif model_name == "videomae-small-finetuned-ssv2":
						lowerCAmelCase_    :Union[str, Any]			     =  torch.Size([1, 1_7_4]		)
						lowerCAmelCase_    :str			     =  torch.tensor([0.2671, -0.4689, -0.8235]		)
			elif model_name == "videomae-base":
						lowerCAmelCase_    :Union[str, Any]			     =  torch.Size([1, 1_4_0_8, 1_5_3_6]		)
						lowerCAmelCase_    :List[str]			     =  torch.tensor([[0.7739, 0.7968, 0.7089], [0.6701, 0.7487, 0.6209], [0.4287, 0.5158, 0.4773]]		)
			elif model_name == "videomae-base-short":
						lowerCAmelCase_    :Dict			     =  torch.Size([1, 1_4_0_8, 1_5_3_6]		)
						lowerCAmelCase_    :List[Any]			     =  torch.tensor([[0.7994, 0.9612, 0.8508], [0.7401, 0.8958, 0.8302], [0.5862, 0.7468, 0.7325]]		)
						# we verified the loss both for normalized and unnormalized targets for this one
						lowerCAmelCase_    :Union[str, Any]			     =  torch.tensor([0.5142]		) if config.norm_pix_loss else torch.tensor([0.6469]		)
			elif model_name == "videomae-large":
						lowerCAmelCase_    :List[Any]			     =  torch.Size([1, 1_4_0_8, 1_5_3_6]		)
						lowerCAmelCase_    :Union[str, Any]			     =  torch.tensor([[0.7149, 0.7997, 0.6966], [0.6768, 0.7869, 0.6948], [0.5139, 0.6221, 0.5605]]		)
			elif model_name == "videomae-large-finetuned-kinetics":
						lowerCAmelCase_    :Optional[Any]			     =  torch.Size([1, 4_0_0]		)
						lowerCAmelCase_    :Union[str, Any]			     =  torch.tensor([0.0771, 0.0011, -0.3625]		)
			elif model_name == "videomae-huge-finetuned-kinetics":
						lowerCAmelCase_    :List[str]			     =  torch.Size([1, 4_0_0]		)
						lowerCAmelCase_    :Dict			     =  torch.tensor([0.2433, 0.1632, -0.4894]		)
			elif model_name == "videomae-base-short-finetuned-kinetics":
						lowerCAmelCase_    :str			     =  torch.Size([1, 4_0_0]		)
						lowerCAmelCase_    :Optional[Any]			     =  torch.tensor([0.6588, 0.0990, -0.2493]		)
			elif model_name == "videomae-base-finetuned-kinetics":
						lowerCAmelCase_    :Any			     =  torch.Size([1, 4_0_0]		)
						lowerCAmelCase_    :Optional[Any]			     =  torch.tensor([0.3669, -0.0688, -0.2421]		)
			elif model_name == "videomae-base-short-ssv2":
						lowerCAmelCase_    :Tuple			     =  torch.Size([1, 1_4_0_8, 1_5_3_6]		)
						lowerCAmelCase_    :Tuple			     =  torch.tensor([[0.4712, 0.5296, 0.5786], [0.2278, 0.2729, 0.4026], [0.0352, 0.0730, 0.2506]]		)
			elif model_name == "videomae-base-short-finetuned-ssv2":
						lowerCAmelCase_    :List[Any]			     =  torch.Size([1, 1_7_4]		)
						lowerCAmelCase_    :List[str]			     =  torch.tensor([-0.0537, -0.1539, -0.3266]		)
			elif model_name == "videomae-base-ssv2":
						lowerCAmelCase_    :Tuple			     =  torch.Size([1, 1_4_0_8, 1_5_3_6]		)
						lowerCAmelCase_    :int			     =  torch.tensor([[0.8131, 0.8727, 0.8546], [0.7366, 0.9377, 0.8870], [0.5935, 0.8874, 0.8564]]		)
			elif model_name == "videomae-base-finetuned-ssv2":
						lowerCAmelCase_    :int			     =  torch.Size([1, 1_7_4]		)
						lowerCAmelCase_    :Union[str, Any]			     =  torch.tensor([0.1961, -0.8337, -0.6389]		)
			else:
						raise ValueError(f"""Model name not supported. Should be one of {model_names}"""		)

			# verify logits
			assert logits.shape == expected_shape
			if "finetuned" in model_name:
						assert torch.allclose(logits[0, :3]      ,     __A      ,     atol=1E-4		)
			else:
						print("""Logits:"""      ,     logits[0, :3, :3]		)
						assert torch.allclose(logits[0, :3, :3]      ,     __A      ,     atol=1E-4		)
			print("""Logits ok!"""		)

			# verify loss, if applicable
			if model_name == "videomae-base-short":
						lowerCAmelCase_    :List[Any]			     =  outputs.loss
						assert torch.allclose(__A      ,     __A      ,     atol=1E-4		)
						print("""Loss ok!"""		)

			if pytorch_dump_folder_path is not None:
						print(f"""Saving model and image processor to {pytorch_dump_folder_path}"""		)
						image_processor.save_pretrained(__A		)
						model.save_pretrained(__A		)

			if push_to_hub:
						print("""Pushing to the hub..."""		)
						model.push_to_hub(__A      ,     organization="""nielsr"""		)


if __name__ == "__main__":
				__UpperCAmelCase       =  argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--checkpoint_url',
				    default='https://drive.google.com/u/1/uc?id=1tEhLyskjb755TJ65ptsrafUG2llSwQE1&amp;export=download&amp;confirm=t&amp;uuid=aa3276eb-fb7e-482a-adec-dc7171df14c4',
				    type=str,
				    help=(
				        'URL of the original PyTorch checkpoint (on Google Drive) you\'d like to convert. Should be a direct'
				        ' download link.'
				    ),
				)
				parser.add_argument(
				    '--pytorch_dump_folder_path',
				    default='/Users/nielsrogge/Documents/VideoMAE/Test',
				    type=str,
				    help='Path to the output PyTorch model directory.',
				)
				parser.add_argument('--model_name', default='videomae-base', type=str, help='Name of the model.')
				parser.add_argument(
				    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
				)

				__UpperCAmelCase       =  parser.parse_args()
				convert_videomae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub)












import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
    snake_case_ : Optional[Any]    		=       "pt"
elif is_tf_available():
    snake_case_ : Union[str, Any]    		=       "tf"
else:
    snake_case_ : str    		=       "jax"







class __snake_case			(			a ,  unittest.TestCase		):
   UpperCAmelCase__  :    List[Any]          =  ByTaTokenizer
   UpperCAmelCase__  :    int          =  False







   def lowerCamelCase						(      self		:						Optional[int]):

        """simple docstring"""


        super().setUp()
        UpperCAmelCase_          =							ByTaTokenizer()
        tokenizer.save_pretrained(self.tmpdirname)







   @cached_property
   def lowerCamelCase						(      self		:						Tuple):

        """simple docstring"""


        return ByTaTokenizer.from_pretrained('''google/byt5-small''')







   def lowerCamelCase						(      self		:						List[str]					,						**_snake_case		:						Union[str, Any]):

        """simple docstring"""


        return self.tokenizer_class.from_pretrained(self.tmpdirname					,						**_snake_case)







   def lowerCamelCase						(      self		:						Dict					,						_snake_case		:						int					,						_snake_case		:						Tuple=False					,						_snake_case		:						Dict=20					,						_snake_case		:						Optional[Any]=5):

        """simple docstring"""


        UpperCAmelCase_          =							[]
        for i in range(len(_snake_case)):
             try:
                  UpperCAmelCase_          =							tokenizer.decode([i]					,						clean_up_tokenization_spaces=_snake_case)
             except UnicodeDecodeError:
                  pass
             toks.append((i, tok))

        UpperCAmelCase_          =							list(filter(lambda _snake_case: re.match(r'''^[ a-zA-Z]+$'''					,						t[1])					,						_snake_case))
        UpperCAmelCase_          =							list(filter(lambda _snake_case: [t[0]] == tokenizer.encode(t[1]					,						add_special_tokens=_snake_case)					,						_snake_case))
        if max_length is not None and len(_snake_case) > max_length:
             UpperCAmelCase_          =							toks[:max_length]
        if min_length is not None and len(_snake_case) < min_length and len(_snake_case) > 0:
             while len(_snake_case) < min_length:
                  UpperCAmelCase_          =							toks + toks
        # toks_str = [t[1] for t in toks]
        UpperCAmelCase_          =							[t[0] for t in toks]

        # Ensure consistency
        UpperCAmelCase_          =							tokenizer.decode(_snake_case					,						clean_up_tokenization_spaces=_snake_case)
        if " " not in output_txt and len(_snake_case) > 1:
             UpperCAmelCase_          =							(
                 tokenizer.decode([toks_ids[0]]					,						clean_up_tokenization_spaces=_snake_case)
                 + ''' '''
                 + tokenizer.decode(toks_ids[1:]					,						clean_up_tokenization_spaces=_snake_case)
             )
        if with_prefix_space:
             UpperCAmelCase_          =							''' ''' + output_txt
        UpperCAmelCase_          =							tokenizer.encode(_snake_case					,						add_special_tokens=_snake_case)
        return output_txt, output_ids







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>'''])
        UpperCAmelCase_          =							tokenizer(['''hi''', '''I went to the gym''', ''''''])
        self.assertListEqual(batch_with_eos_added['''input_ids''']					,						batch_without_eos_added['''input_ids'''])







   def lowerCamelCase						(      self		:						str):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							'''Unicode €.'''
        UpperCAmelCase_          =							tokenizer(_snake_case)
        UpperCAmelCase_          =							[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
        self.assertEqual(encoded['''input_ids''']					,						_snake_case)

        # decoding
        UpperCAmelCase_          =							tokenizer.decode(_snake_case)
        self.assertEqual(_snake_case					,						'''Unicode €.</s>''')

        UpperCAmelCase_          =							tokenizer('''e è é ê ë''')
        UpperCAmelCase_          =							[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
        self.assertEqual(encoded['''input_ids''']					,						_snake_case)
        # decoding
        UpperCAmelCase_          =							tokenizer.decode(_snake_case)
        self.assertEqual(_snake_case					,						'''e è é ê ë</s>''')

        # encode/decode, but with `encode` instead of `__call__`
        self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''))					,						'''e è é ê ë</s>''')







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
        # fmt: off
        UpperCAmelCase_          =							[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
        # fmt: on
        UpperCAmelCase_          =							tokenizer(_snake_case					,						padding=_snake_case					,						return_tensors=_snake_case)
        self.assertIsInstance(_snake_case					,						_snake_case)

        if FRAMEWORK != "jax":
             UpperCAmelCase_          =							list(batch.input_ids.numpy()[0])
        else:
             UpperCAmelCase_          =							list(batch.input_ids.tolist()[0])

        self.assertListEqual(_snake_case					,						_snake_case)

        self.assertEqual((2, 37)					,						batch.input_ids.shape)
        self.assertEqual((2, 37)					,						batch.attention_mask.shape)







   def lowerCamelCase						(      self		:						Optional[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
        UpperCAmelCase_          =							tokenizer(_snake_case					,						padding=_snake_case					,						return_tensors=_snake_case)
        # check if input_ids are returned and no decoder_input_ids
        self.assertIn('''input_ids'''					,						_snake_case)
        self.assertIn('''attention_mask'''					,						_snake_case)
        self.assertNotIn('''decoder_input_ids'''					,						_snake_case)
        self.assertNotIn('''decoder_attention_mask'''					,						_snake_case)







   def lowerCamelCase						(      self		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							[
            '''Summary of the text.''',
            '''Another summary.''',
        ]
        UpperCAmelCase_          =							tokenizer(
            text_target=_snake_case					,						max_length=32					,						padding='''max_length'''					,						truncation=_snake_case					,						return_tensors=_snake_case)
        self.assertEqual(32					,						targets['''input_ids'''].shape[1])







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        UpperCAmelCase_          =							self.ta_base_tokenizer
        UpperCAmelCase_          =							['''A long paragraph for summarization. </s>''']
        UpperCAmelCase_          =							['''Summary of the text. </s>''']
        # fmt: off
        UpperCAmelCase_          =							[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
        UpperCAmelCase_          =							[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
        # fmt: on

        UpperCAmelCase_          =							tokenizer(_snake_case					,						text_target=_snake_case)

        self.assertEqual(_snake_case					,						batch['''input_ids'''][0])
        self.assertEqual(_snake_case					,						batch['''labels'''][0])







   def lowerCamelCase						(      self		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                  self.assertNotEqual(tokenizer.model_max_length					,						42)

        # Now let's start the test
        UpperCAmelCase_          =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                  # Isolate this from the other tests because we save additional tokens/etc
                  UpperCAmelCase_          =							tempfile.mkdtemp()

                  UpperCAmelCase_          =							''' He is very happy, UNwant\u00E9d,running'''
                  UpperCAmelCase_          =							tokenizer.encode(_snake_case					,						add_special_tokens=_snake_case)
                  tokenizer.save_pretrained(_snake_case)

                  UpperCAmelCase_          =							tokenizer.__class__.from_pretrained(_snake_case)
                  UpperCAmelCase_          =							after_tokenizer.encode(_snake_case					,						add_special_tokens=_snake_case)
                  self.assertListEqual(_snake_case					,						_snake_case)

                  shutil.rmtree(_snake_case)

        UpperCAmelCase_          =							self.get_tokenizers(model_max_length=42)
        for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                  # Isolate this from the other tests because we save additional tokens/etc
                  UpperCAmelCase_          =							tempfile.mkdtemp()

                  UpperCAmelCase_          =							''' He is very happy, UNwant\u00E9d,running'''
                  tokenizer.add_tokens(['''bim''', '''bambam'''])
                  UpperCAmelCase_          =							tokenizer.additional_special_tokens
                  additional_special_tokens.append('''new_additional_special_token''')
                  tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens})
                  UpperCAmelCase_          =							tokenizer.encode(_snake_case					,						add_special_tokens=_snake_case)
                  tokenizer.save_pretrained(_snake_case)

                  UpperCAmelCase_          =							tokenizer.__class__.from_pretrained(_snake_case)
                  UpperCAmelCase_          =							after_tokenizer.encode(_snake_case					,						add_special_tokens=_snake_case)
                  self.assertListEqual(_snake_case					,						_snake_case)
                  self.assertIn('''new_additional_special_token'''					,						after_tokenizer.additional_special_tokens)
                  self.assertEqual(after_tokenizer.model_max_length					,						42)

                  UpperCAmelCase_          =							tokenizer.__class__.from_pretrained(_snake_case					,						model_max_length=43)
                  self.assertEqual(tokenizer.model_max_length					,						43)

                  shutil.rmtree(_snake_case)







   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        UpperCAmelCase_          =							[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(_snake_case)

                  with open(os.path.join(_snake_case					,						'''special_tokens_map.json''')					,						encoding='''utf-8''') as json_file:
                       UpperCAmelCase_          =							json.load(_snake_case)

                  with open(os.path.join(_snake_case					,						'''tokenizer_config.json''')					,						encoding='''utf-8''') as json_file:
                       UpperCAmelCase_          =							json.load(_snake_case)

                  UpperCAmelCase_          =							[F"""<extra_id_{i}>""" for i in range(125)]

                  UpperCAmelCase_          =							added_tokens_extra_ids + [
                      '''an_additional_special_token'''
                  ]
                  UpperCAmelCase_          =							added_tokens_extra_ids + [
                      '''an_additional_special_token'''
                  ]

                  with open(os.path.join(_snake_case					,						'''special_tokens_map.json''')					,						'''w'''					,						encoding='''utf-8''') as outfile:
                       json.dump(_snake_case					,						_snake_case)
                  with open(os.path.join(_snake_case					,						'''tokenizer_config.json''')					,						'''w'''					,						encoding='''utf-8''') as outfile:
                       json.dump(_snake_case					,						_snake_case)

                  # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                  # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                  # "special_tokens_map.json" files
                  UpperCAmelCase_          =							tokenizer_class.from_pretrained(
                      _snake_case					,						)
                  self.assertIn(
                      '''an_additional_special_token'''					,						tokenizer_without_change_in_init.additional_special_tokens)
                  # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                  self.assertEqual(
                      ['''an_additional_special_token''']					,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                          tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']))					,						)

                  # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                  UpperCAmelCase_          =							added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''					,						lstrip=_snake_case)]
                  UpperCAmelCase_          =							tokenizer_class.from_pretrained(
                      _snake_case					,						additional_special_tokens=_snake_case					,						)

                  self.assertIn('''a_new_additional_special_token'''					,						tokenizer.additional_special_tokens)
                  self.assertEqual(
                      ['''a_new_additional_special_token''']					,						tokenizer.convert_ids_to_tokens(
                          tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']))					,						)







   def lowerCamelCase						(      self		:						Any):

        """simple docstring"""


        UpperCAmelCase_          =							[]
        if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

        if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

        for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                  tokenizer_utils.save_pretrained(_snake_case)

                  UpperCAmelCase_          =							tokenizer_class.from_pretrained(_snake_case)

                  self.assertTrue(tokenizer.decode([255]) == '''''')







   def lowerCamelCase						(      self		:						int):

        """simple docstring"""


        pass







   def lowerCamelCase						(      self		:						Optional[int]):

        """simple docstring"""


        pass







   def lowerCamelCase						(      self		:						Dict):

        """simple docstring"""


        pass







   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        pass







   def lowerCamelCase						(      self		:						Tuple):

        """simple docstring"""


        UpperCAmelCase_          =							self.get_tokenizers(fast=_snake_case					,						do_lower_case=_snake_case)
        for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                  UpperCAmelCase_          =							['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
                  UpperCAmelCase_          =							tokenizer.convert_tokens_to_string(_snake_case)

                  self.assertIsInstance(_snake_case					,						_snake_case)







   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        UpperCAmelCase_          =							self.get_tokenizers()
        for tokenizer in tokenizers:
             with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                  UpperCAmelCase_          =							[
                      '''bos_token''',
                      '''eos_token''',
                      '''unk_token''',
                      '''sep_token''',
                      '''pad_token''',
                      '''cls_token''',
                      '''mask_token''',
                  ]

                  UpperCAmelCase_          =							0
                  UpperCAmelCase_          =							tokenizer.convert_ids_to_tokens(
                      _snake_case					,						skip_special_tokens=_snake_case)

                  for attr in attributes_list:
                       setattr(_snake_case					,						attr + '''_id'''					,						_snake_case)
                       self.assertEqual(getattr(_snake_case					,						_snake_case)					,						_snake_case)
                       self.assertEqual(getattr(_snake_case					,						attr + '''_id''')					,						_snake_case)

                       setattr(_snake_case					,						attr + '''_id'''					,						_snake_case)
                       self.assertEqual(getattr(_snake_case					,						_snake_case)					,						_snake_case)
                       self.assertEqual(getattr(_snake_case					,						attr + '''_id''')					,						_snake_case)

                  setattr(_snake_case					,						'''additional_special_tokens_ids'''					,						[])
                  self.assertListEqual(getattr(_snake_case					,						'''additional_special_tokens''')					,						[])
                  self.assertListEqual(getattr(_snake_case					,						'''additional_special_tokens_ids''')					,						[])

                  setattr(_snake_case					,						'''additional_special_tokens_ids'''					,						[token_id_to_test_setters])
                  self.assertListEqual(getattr(_snake_case					,						'''additional_special_tokens''')					,						[token_to_test_setters])
                  self.assertListEqual(getattr(_snake_case					,						'''additional_special_tokens_ids''')					,						[token_id_to_test_setters])






