




'''simple docstring'''

import argparse

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    CLIPTokenizer,
    CLIPTokenizerFast,
    VideoMAEImageProcessor,
    XCLIPConfig,
    XCLIPModel,
    XCLIPProcessor,
    XCLIPTextConfig,
    XCLIPVisionConfig,
)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					Dict:

					'''simple docstring'''

					snake_case_									=				XCLIPTextConfig()

					# derive patch size from model name
					snake_case_									=				model_name.find('''patch'''	)
					snake_case_									=				int(model_name[start_idx + len('''patch'''	) : start_idx + len('''patch'''	) + 2]	)
					snake_case_									=				XCLIPVisionConfig(patch_size=__UpperCAmelCase,      num_frames=__UpperCAmelCase	)

					if "large" in model_name:
										snake_case_									=				768
										snake_case_									=				3072
										snake_case_									=				12

										snake_case_									=				1024
										snake_case_									=				4096
										snake_case_									=				16
										snake_case_									=				24
										snake_case_									=				768
										snake_case_									=				3072

					if model_name == "xclip-large-patch14-16-frames":
										snake_case_									=				336

					snake_case_									=				XCLIPConfig.from_text_vision_configs(__UpperCAmelCase,      __UpperCAmelCase	)

					if "large" in model_name:
										snake_case_									=				768

					return config


def 		__magic_name__    (	__UpperCAmelCase	)		->					List[Any]:

					'''simple docstring'''

					if name == "token_embedding.weight":
										snake_case_									=				name.replace('''token_embedding.weight''',      '''text_model.embeddings.token_embedding.weight'''	)
					if name == "positional_embedding":
										snake_case_									=				name.replace('''positional_embedding''',      '''text_model.embeddings.position_embedding.weight'''	)
					if "ln_1" in name:
										snake_case_									=				name.replace('''ln_1''',      '''layer_norm1'''	)
					if "ln_2" in name:
										snake_case_									=				name.replace('''ln_2''',      '''layer_norm2'''	)
					if "c_fc" in name:
										snake_case_									=				name.replace('''c_fc''',      '''fc1'''	)
					if "c_proj" in name:
										snake_case_									=				name.replace('''c_proj''',      '''fc2'''	)
					if name.startswith('''transformer.resblocks'''	):
										snake_case_									=				name.replace('''transformer.resblocks''',      '''text_model.encoder.layers'''	)
					if "attn.out_proj" in name and "message" not in name:
										snake_case_									=				name.replace('''attn.out_proj''',      '''self_attn.out_proj'''	)
					if "ln_final" in name:
										snake_case_									=				name.replace('''ln_final''',      '''text_model.final_layer_norm'''	)
					# visual encoder
					if name == "visual.class_embedding":
										snake_case_									=				name.replace('''visual.class_embedding''',      '''vision_model.embeddings.class_embedding'''	)
					if name == "visual.positional_embedding":
										snake_case_									=				name.replace('''visual.positional_embedding''',      '''vision_model.embeddings.position_embedding.weight'''	)
					if name.startswith('''visual.transformer.resblocks'''	):
										snake_case_									=				name.replace('''visual.transformer.resblocks''',      '''vision_model.encoder.layers'''	)
					if "visual.conv1" in name:
										snake_case_									=				name.replace('''visual.conv1''',      '''vision_model.embeddings.patch_embedding'''	)
					if "visual.ln_pre" in name:
										snake_case_									=				name.replace('''visual.ln_pre''',      '''vision_model.pre_layernorm'''	)
					if "visual.ln_post" in name:
										snake_case_									=				name.replace('''visual.ln_post''',      '''vision_model.post_layernorm'''	)
					if "visual.proj" in name:
										snake_case_									=				name.replace('''visual.proj''',      '''visual_projection.weight'''	)
					if "text_projection" in name:
										snake_case_									=				name.replace('''text_projection''',      '''text_projection.weight'''	)
					# things on top
					if "prompts_visual_proj" in name:
										snake_case_									=				name.replace('''prompts_visual_proj''',      '''prompts_visual_projection'''	)
					if "prompts_visual_ln" in name:
										snake_case_									=				name.replace('''prompts_visual_ln''',      '''prompts_visual_layernorm'''	)
					# mit
					if name == "mit.positional_embedding":
										snake_case_									=				name.replace('''positional''',      '''position'''	)
					if name.startswith('''mit.resblocks'''	):
										snake_case_									=				name.replace('''mit.resblocks''',      '''mit.encoder.layers'''	)
					# prompts generator
					if name.startswith('''prompts_generator.norm'''	):
										snake_case_									=				name.replace('''prompts_generator.norm''',      '''prompts_generator.layernorm'''	)

					return name


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					Optional[Any]:

					'''simple docstring'''

					for key in orig_state_dict.copy().keys():
										snake_case_									=				orig_state_dict.pop(__UpperCAmelCase	)

										if "attn.in_proj" in key:
															snake_case_									=				key.split('''.'''	)
															if key.startswith('''visual'''	):
																				snake_case_									=				key_split[3]
																				snake_case_									=				config.vision_config.hidden_size
																				if "message_attn" in key:
																									if "weight" in key:
																														snake_case_									=				val[
																														    :dim, :
																														]
																														snake_case_									=				val[
																														    dim : dim * 2, :
																														]
																														snake_case_									=				val[
																														    -dim:, :
																														]
																									else:
																														snake_case_									=				val[
																														    :dim
																														]
																														snake_case_									=				val[
																														    dim : dim * 2
																														]
																														snake_case_									=				val[
																														    -dim:
																														]
																				else:
																									if "weight" in key:
																														snake_case_									=				val[
																														    :dim, :
																														]
																														snake_case_									=				val[
																														    dim : dim * 2, :
																														]
																														snake_case_									=				val[
																														    -dim:, :
																														]
																									else:
																														snake_case_									=				val[:dim]
																														snake_case_									=				val[
																														    dim : dim * 2
																														]
																														snake_case_									=				val[-dim:]
															elif key.startswith('''mit'''	):
																				snake_case_									=				key_split[2]
																				snake_case_									=				config.vision_config.mit_hidden_size
																				if "weight" in key:
																									snake_case_									=				val[:dim, :]
																									snake_case_									=				val[dim : dim * 2, :]
																									snake_case_									=				val[-dim:, :]
																				else:
																									snake_case_									=				val[:dim]
																									snake_case_									=				val[dim : dim * 2]
																									snake_case_									=				val[-dim:]
															else:
																				snake_case_									=				key_split[2]
																				snake_case_									=				config.text_config.hidden_size
																				if "weight" in key:
																									snake_case_									=				val[:dim, :]
																									snake_case_									=				val[
																									    dim : dim * 2, :
																									]
																									snake_case_									=				val[-dim:, :]
																				else:
																									snake_case_									=				val[:dim]
																									snake_case_									=				val[
																									    dim : dim * 2
																									]
																									snake_case_									=				val[-dim:]
										else:
															snake_case_									=				rename_key(__UpperCAmelCase	)
															if new_key_name in ["visual_projection.weight", "text_projection.weight"]:
																				snake_case_									=				val.T
															snake_case_									=				val

					return orig_state_dict


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					if num_frames == 8:
										snake_case_									=				'''eating_spaghetti_8_frames.npy'''
					elif num_frames == 16:
										snake_case_									=				'''eating_spaghetti.npy'''
					elif num_frames == 32:
										snake_case_									=				'''eating_spaghetti_32_frames.npy'''
					snake_case_									=				hf_hub_download(
					    repo_id='''hf-internal-testing/spaghetti-video''',      filename=__UpperCAmelCase,      repo_type='''dataset''',      )
					snake_case_									=				np.load(__UpperCAmelCase	)
					return list(__UpperCAmelCase	)


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase=None,      __UpperCAmelCase=False	)		->					List[str]:

					'''simple docstring'''

					snake_case_									=				{
					    # fully supervised kinetics-400 checkpoints
					    '''xclip-base-patch32''': '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_8.pth''',
					    '''xclip-base-patch32-16-frames''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_16.pth'''
					    ),
					    '''xclip-base-patch16''': '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_8.pth''',
					    '''xclip-base-patch16-16-frames''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_16.pth'''
					    ),
					    '''xclip-large-patch14''': '''https://drive.google.com/u/0/uc?id=1NUOImq0o5DlQTST17iIP3vG7DgmHQuCx&amp;export=download&amp;confirm=t&amp;uuid=b26caedc-88e2-473e-830a-9d158b653cdb''',
					    '''xclip-large-patch14-16-frames''': '''https://drive.google.com/u/0/uc?id=1FOYgnJc097OJ4lGwtRCCydQyVPJEOH7d&amp;export=download&amp;confirm=t&amp;uuid=538fa810-e671-4050-b385-9a623f89804f''',
					    # fully supervised kinetics-600 checkpoints
					    '''xclip-base-patch16-kinetics-600''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_8.pth'''
					    ),
					    '''xclip-base-patch16-kinetics-600-16-frames''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_16.pth'''
					    ),
					    '''xclip-large-patch14-kinetics-600''': '''https://drive.google.com/u/0/uc?id=1FV8C1INuM91sLAN4ImjzePLIlpMSihwV&amp;export=download&amp;confirm=t&amp;uuid=141d4977-4a65-44ae-864f-4b0c19f838be''',
					    # few shot
					    '''xclip-base-patch16-hmdb-2-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_2.pth'''
					    ),
					    '''xclip-base-patch16-hmdb-4-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_4.pth'''
					    ),
					    '''xclip-base-patch16-hmdb-8-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_8.pth'''
					    ),
					    '''xclip-base-patch16-hmdb-16-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_16.pth'''
					    ),
					    '''xclip-base-patch16-ucf-2-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_2.pth'''
					    ),
					    '''xclip-base-patch16-ucf-4-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_4.pth'''
					    ),
					    '''xclip-base-patch16-ucf-8-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_8.pth'''
					    ),
					    '''xclip-base-patch16-ucf-16-shot''': (
					        '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_16.pth'''
					    ),
					    # zero shot
					    '''xclip-base-patch16-zero-shot''': '''https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/zero.pth''',
					}

					snake_case_									=				model_to_url[model_name]
					snake_case_									=				8
					if "16-frames" in model_name:
										snake_case_									=				16
					elif "shot" in model_name:
										snake_case_									=				32

					snake_case_									=				get_xclip_config(__UpperCAmelCase,      __UpperCAmelCase	)
					snake_case_									=				XCLIPModel(__UpperCAmelCase	)
					model.eval()

					if "drive" in checkpoint_url:
										snake_case_									=				'''pytorch_model.bin'''
										gdown.cached_download(__UpperCAmelCase,      __UpperCAmelCase,      quiet=__UpperCAmelCase	)
										snake_case_									=				torch.load(__UpperCAmelCase,      map_location='''cpu'''	)['''model''']
					else:
										snake_case_									=				torch.hub.load_state_dict_from_url(__UpperCAmelCase	)['''model''']

					snake_case_									=				convert_state_dict(__UpperCAmelCase,      __UpperCAmelCase	)

					snake_case_									=				XCLIPModel(__UpperCAmelCase	)
					snake_case_			,snake_case_									=				model.load_state_dict(__UpperCAmelCase,      strict=__UpperCAmelCase	)
					assert missing_keys == ["text_model.embeddings.position_ids", "vision_model.embeddings.position_ids"]
					model.eval()

					snake_case_									=				336 if model_name == '''xclip-large-patch14-16-frames''' else 224
					snake_case_									=				VideoMAEImageProcessor(size=__UpperCAmelCase	)
					snake_case_									=				CLIPTokenizer.from_pretrained('''openai/clip-vit-base-patch32'''	)
					snake_case_									=				CLIPTokenizerFast.from_pretrained('''openai/clip-vit-base-patch32'''	)
					snake_case_									=				XCLIPProcessor(image_processor=__UpperCAmelCase,      tokenizer=__UpperCAmelCase	)

					snake_case_									=				prepare_video(__UpperCAmelCase	)
					snake_case_									=				processor(
					    text=['''playing sports''', '''eating spaghetti''', '''go shopping'''],      videos=__UpperCAmelCase,      return_tensors='''pt''',      padding=__UpperCAmelCase	)

					print('''Shape of pixel values:''',      inputs.pixel_values.shape	)

					with torch.no_grad():
										snake_case_									=				model(**__UpperCAmelCase	)

					# Verify outputs
					snake_case_									=				outputs.logits_per_video
					snake_case_									=				logits_per_video.softmax(dim=1	)
					print('''Probs:''',      __UpperCAmelCase	)
					# kinetics-400
					if model_name == "xclip-base-patch32":
										snake_case_									=				torch.tensor([[0.0_0_1_9, 0.9_9_5_1, 0.0_0_3_0]]	)
					elif model_name == "xclip-base-patch32-16-frames":
										snake_case_									=				torch.tensor([[7.0999e-04, 9.9883e-01, 4.5580e-04]]	)
					elif model_name == "xclip-base-patch16":
										snake_case_									=				torch.tensor([[0.0_0_8_3, 0.9_6_8_1, 0.0_2_3_6]]	)
					elif model_name == "xclip-base-patch16-16-frames":
										snake_case_									=				torch.tensor([[7.6937e-04, 9.9728e-01, 1.9473e-03]]	)
					elif model_name == "xclip-large-patch14":
										snake_case_									=				torch.tensor([[0.0_0_6_2, 0.9_8_6_4, 0.0_0_7_5]]	)
					elif model_name == "xclip-large-patch14-16-frames":
										snake_case_									=				torch.tensor([[3.3877e-04, 9.9937e-01, 2.8888e-04]]	)
					# kinetics-600
					elif model_name == "xclip-base-patch16-kinetics-600":
										snake_case_									=				torch.tensor([[0.0_5_5_5, 0.8_9_1_4, 0.0_5_3_1]]	)
					elif model_name == "xclip-base-patch16-kinetics-600-16-frames":
										snake_case_									=				torch.tensor([[3.8554e-04, 9.9929e-01, 3.2754e-04]]	)
					elif model_name == "xclip-large-patch14-kinetics-600":
										snake_case_									=				torch.tensor([[0.0_0_3_6, 0.9_9_2_0, 0.0_0_4_5]]	)
					# few shot
					elif model_name == "xclip-base-patch16-hmdb-2-shot":
										snake_case_									=				torch.tensor([[7.1890e-06, 9.9994e-01, 5.6559e-05]]	)
					elif model_name == "xclip-base-patch16-hmdb-4-shot":
										snake_case_									=				torch.tensor([[1.0320e-05, 9.9993e-01, 6.2435e-05]]	)
					elif model_name == "xclip-base-patch16-hmdb-8-shot":
										snake_case_									=				torch.tensor([[4.1377e-06, 9.9990e-01, 9.8386e-05]]	)
					elif model_name == "xclip-base-patch16-hmdb-16-shot":
										snake_case_									=				torch.tensor([[4.1347e-05, 9.9962e-01, 3.3411e-04]]	)
					elif model_name == "xclip-base-patch16-ucf-2-shot":
										snake_case_									=				torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]	)
					elif model_name == "xclip-base-patch16-ucf-4-shot":
										snake_case_									=				torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]	)
					elif model_name == "xclip-base-patch16-ucf-8-shot":
										snake_case_									=				torch.tensor([[0.0_0_2_7, 0.9_9_0_4, 0.0_0_7_0]]	)
					elif model_name == "xclip-base-patch16-ucf-16-shot":
										snake_case_									=				torch.tensor([[9.8219e-04, 9.9593e-01, 3.0863e-03]]	)
					# zero shot
					elif model_name == "xclip-base-patch16-zero-shot":
										snake_case_									=				torch.tensor([[3.5082e-04, 9.9785e-01, 1.7966e-03]]	)
					else:
										raise ValueError(F"Model name {model_name} not supported"	)
					assert torch.allclose(__UpperCAmelCase,      __UpperCAmelCase,      atol=1e-3	)
					print('''Looks ok!'''	)

					if pytorch_dump_folder_path is not None:
										print(F"Saving model {model_name} to {pytorch_dump_folder_path}"	)
										model.save_pretrained(__UpperCAmelCase	)

					if push_to_hub:
										print('''Pushing model, processor and slow tokenizer files to the hub...'''	)
										model.push_to_hub(__UpperCAmelCase,      organization='''nielsr'''	)
										processor.push_to_hub(__UpperCAmelCase,      organization='''nielsr'''	)
										slow_tokenizer.push_to_hub(__UpperCAmelCase,      organization='''nielsr'''	)


if __name__ == "__main__":
							a  :				Tuple									=  argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--model_name',
							    default='xclip-base-patch32',
							    type=str,
							    help='Name of the model.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
							)
							parser.add_argument(
							    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
							)

							a  :				Optional[int]									=  parser.parse_args()
							convert_xclip_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







'''simple docstring'''

import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_ta import TaTokenizer
else:
							a  :				Dict									=  None


a  :				List[Any]									=  logging.get_logger(__name__)

a  :				List[Any]									=  {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a  :				str									=  {
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
a  :				List[Any]									=  {
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}






class 						a				(   _lowerCamelCase     ):
				snake_case_      =		VOCAB_FILES_NAMES
				snake_case_      =		PRETRAINED_VOCAB_FILES_MAP
				snake_case_      =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				snake_case_      =		["input_ids", "attention_mask"]
				snake_case_      =		TaTokenizer

				snake_case_      =		[]


				def __init__(		self    :					List[Any]       ,				lowercase_    :					int=None       ,				lowercase_    :					Dict=None       ,				lowercase_    :					Dict="</s>"       ,				lowercase_    :					List[Any]="<unk>"       ,				lowercase_    :					int="<pad>"       ,				lowercase_    :					int=100       ,				lowercase_    :					List[Any]=None       ,				**lowercase_    :					List[str]       ,				):
									# Add extra_ids to the special token list
									if extra_ids > 0 and additional_special_tokens is None:
														snake_case_									=				[F"<extra_id_{i}>" for i in range(lowercase_					)]
									elif extra_ids > 0 and additional_special_tokens is not None:
														# Check that we have the right number of extra special tokens
														snake_case_									=				len(set(filter(lambda lowercase_					: bool('''extra_id_''' in str(lowercase_					)					)       ,				lowercase_					)					)					)
														if extra_tokens != extra_ids:
																			raise ValueError(
																			    F"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
																			    ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
																			    ''' tokens'''					)

									super().__init__(
									    lowercase_       ,				tokenizer_file=lowercase_       ,				eos_token=lowercase_       ,				unk_token=lowercase_       ,				pad_token=lowercase_       ,				extra_ids=lowercase_       ,				additional_special_tokens=lowercase_       ,				**lowercase_       ,				)

									snake_case_									=				vocab_file
									snake_case_									=				False if not self.vocab_file else True
									snake_case_									=				extra_ids


				@staticmethod
				def   A_ (		lowercase_    :					Optional[int]       ,				lowercase_    :					List[str]       ,				lowercase_    :					int					):
									if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
														snake_case_									=				TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
														if init_max_model_length is not None and init_max_model_length != max_model_length:
																			return init_max_model_length
														elif init_max_model_length is None:
																			warnings.warn(
																			    '''This tokenizer was incorrectly instantiated with a model max length of'''
																			    F" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
																			    ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
																			    ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
																			    F" {pretrained_model_name_or_path} automatically truncating your input to"
																			    F" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
																			    F" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
																			    ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
																			    ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,				lowercase_       ,				)

									return max_model_length


				def   A_ (		self    :					Optional[Any]       ,				lowercase_    :					str       ,				lowercase_    :					Optional[str] = None					):
									if not self.can_save_slow_tokenizer:
														raise ValueError(
														    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
														    '''tokenizer.'''					)

									if not os.path.isdir(lowercase_					):
														logger.error(F"Vocabulary path ({save_directory}) should be a directory"					)
														return
									snake_case_									=				os.path.join(
									    lowercase_       ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase_					):
														copyfile(self.vocab_file       ,				lowercase_					)
														logger.info(F"Copy vocab file to {out_vocab_file}"					)

									return (out_vocab_file,)


				def   A_ (		self    :					Optional[int]       ,				lowercase_    :					List[int]       ,				lowercase_    :					Optional[List[int]] = None					):
									snake_case_									=				token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
														return self.prefix_tokens + token_ids_a
									else:
														snake_case_									=				token_ids_a + [self.eos_token_id]
														return self.prefix_tokens + token_ids_a + token_ids_a


				def   A_ (		self    :					int       ,				lowercase_    :					List[int]       ,				lowercase_    :					Optional[List[int]] = None					):
									snake_case_									=				[self.eos_token_id]

									if token_ids_a is None:
														return len(token_ids_a + eos					) * [0]
									return len(token_ids_a + eos + token_ids_a + eos					) * [0]


				def   A_ (		self    :					Dict					):
									return list(
									    set(filter(lambda lowercase_					: bool(re.search(R'''<extra_id_\d+>'''       ,				lowercase_					)					) is not None       ,				self.additional_special_tokens					)					)					)


				def   A_ (		self    :					Any					):
									return [self.convert_tokens_to_ids(lowercase_					) for token in self.get_sentinel_tokens()]

