
"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__      :    List[Any]    							= logging.get_logger(__name__)

UpperCAmelCase__      :    List[Any]    							= {'vocab_file': 'sentencepiece.bpe.model'}

UpperCAmelCase__      :    List[Any]    							= {
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

UpperCAmelCase__      :    Optional[int]    							= {
    'camembert-base': 5_1_2,
}

UpperCAmelCase__      :    List[Any]    							= '▁'



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Optional[int]					       =   VOCAB_FILES_NAMES
				__UpperCamelCase      :						Union[str, Any]					       =   PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase      :						List[Any]					       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase      :						Optional[Any]					       =   ['''input_ids''', '''attention_mask''']



				def __init__(self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__="<s>"   ,       SCREAMING_SNAKE_CASE__="</s>"   ,       SCREAMING_SNAKE_CASE__="</s>"   ,       SCREAMING_SNAKE_CASE__="<s>"   ,       SCREAMING_SNAKE_CASE__="<unk>"   ,       SCREAMING_SNAKE_CASE__="<pad>"   ,       SCREAMING_SNAKE_CASE__="<mask>"   ,       SCREAMING_SNAKE_CASE__=["<s>NOTUSED", "</s>NOTUSED"]   ,       SCREAMING_SNAKE_CASE__ = None   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		None:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else mask_token

											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=SCREAMING_SNAKE_CASE__   ,       eos_token=SCREAMING_SNAKE_CASE__   ,       unk_token=SCREAMING_SNAKE_CASE__   ,       sep_token=SCREAMING_SNAKE_CASE__   ,       cls_token=SCREAMING_SNAKE_CASE__   ,       pad_token=SCREAMING_SNAKE_CASE__   ,       mask_token=SCREAMING_SNAKE_CASE__   ,       additional_special_tokens=SCREAMING_SNAKE_CASE__   ,       sp_model_kwargs=self.sp_model_kwargs   ,       **SCREAMING_SNAKE_CASE__   ,       )
											SCREAMING_SNAKE_CASE__						:       List[str]           =   spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(str(SCREAMING_SNAKE_CASE__    )    )
											SCREAMING_SNAKE_CASE__						:       Dict           =   vocab_file
											# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
											# sentencepiece vocabulary (this is the case for <s> and </s>
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   {"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
											SCREAMING_SNAKE_CASE__						:       Any           =   len(self.fairseq_tokens_to_ids    )
											SCREAMING_SNAKE_CASE__						:       Any           =   len(self.sp_model    ) + len(self.fairseq_tokens_to_ids    )
											SCREAMING_SNAKE_CASE__						:       Any           =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		List[int]:



											"""simple docstring"""




											if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   [self.cls_token_id]
											SCREAMING_SNAKE_CASE__						:       int           =   [self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None   ,       SCREAMING_SNAKE_CASE__ = False    )					->		List[int]:



											"""simple docstring"""




											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=SCREAMING_SNAKE_CASE__   ,       token_ids_a=SCREAMING_SNAKE_CASE__   ,       already_has_special_tokens=SCREAMING_SNAKE_CASE__    )

											if token_ids_a is None:
																		return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]
											return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		List[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   [self.sep_token_id]
											SCREAMING_SNAKE_CASE__						:       Any           =   [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]



				@property
				def 	__magic_name__    (self    )					->		Dict:



											"""simple docstring"""




											return len(self.fairseq_tokens_to_ids    ) + len(self.sp_model    )



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		List[str]:



											"""simple docstring"""




											return self.sp_model.encode(SCREAMING_SNAKE_CASE__   ,       out_type=SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		int:



											"""simple docstring"""




											if token in self.fairseq_tokens_to_ids:
																		return self.fairseq_tokens_to_ids[token]
											elif self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__    ) == 0:
																		# Convert sentence piece unk token to fairseq unk token index
																		return self.unk_token_id
											return self.fairseq_offset + self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Any:



											"""simple docstring"""




											if index in self.fairseq_ids_to_tokens:
																		return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   []
											SCREAMING_SNAKE_CASE__						:       Tuple           =   """"""
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   False
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									if not prev_is_special:
																																out_string += " "
																									out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__    ) + token
																									SCREAMING_SNAKE_CASE__						:       Dict           =   True
																									SCREAMING_SNAKE_CASE__						:       List[str]           =   []
																		else:
																									current_sub_tokens.append(SCREAMING_SNAKE_CASE__    )
																									SCREAMING_SNAKE_CASE__						:       Dict           =   False
											out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE__    )
											return out_string.strip()



				def __getstate__(self    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Any           =   self.__dict__.copy()
											SCREAMING_SNAKE_CASE__						:       Any           =   None
											return state



				def __setstate__(self   ,       SCREAMING_SNAKE_CASE__    )					->		str:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Dict           =   d

											# for backward compatibility
											if not hasattr(self   ,       """sp_model_kwargs"""    ):
																		SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   {}

											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		Tuple[str]:



											"""simple docstring"""




											if not os.path.isdir(SCREAMING_SNAKE_CASE__    ):
																		logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
																		return
											SCREAMING_SNAKE_CASE__						:       int           =   os.path.join(
											    SCREAMING_SNAKE_CASE__   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE__    ) and os.path.isfile(self.vocab_file    ):
																		copyfile(self.vocab_file   ,       SCREAMING_SNAKE_CASE__    )
											elif not os.path.isfile(self.vocab_file    ):
																		with open(SCREAMING_SNAKE_CASE__   ,       """wb"""    ) as fi:
																									SCREAMING_SNAKE_CASE__						:       Optional[int]           =   self.sp_model.serialized_model_proto()
																									fi.write(SCREAMING_SNAKE_CASE__    )

											return (out_vocab_file,)






"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__      :    Optional[Any]    							= logging.get_logger(__name__)

UpperCAmelCase__      :    Dict    							= {
    'MIT/ast-finetuned-audioset-10-10-0.4593': (
        'https://huggingface.co/MIT/ast-finetuned-audioset-10-10-0.4593/resolve/main/config.json'
    ),
}



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Optional[int]					       =   '''audio-spectrogram-transformer'''



				def __init__(self   ,       SCREAMING_SNAKE_CASE__=7_68   ,       SCREAMING_SNAKE_CASE__=12   ,       SCREAMING_SNAKE_CASE__=12   ,       SCREAMING_SNAKE_CASE__=30_72   ,       SCREAMING_SNAKE_CASE__="gelu"   ,       SCREAMING_SNAKE_CASE__=0.0   ,       SCREAMING_SNAKE_CASE__=0.0   ,       SCREAMING_SNAKE_CASE__=0.02   ,       SCREAMING_SNAKE_CASE__=1E-12   ,       SCREAMING_SNAKE_CASE__=16   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=10   ,       SCREAMING_SNAKE_CASE__=10   ,       SCREAMING_SNAKE_CASE__=10_24   ,       SCREAMING_SNAKE_CASE__=1_28   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		Tuple:



											"""simple docstring"""




											super().__init__(**SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   hidden_size
											SCREAMING_SNAKE_CASE__						:       str           =   num_hidden_layers
											SCREAMING_SNAKE_CASE__						:       int           =   num_attention_heads
											SCREAMING_SNAKE_CASE__						:       Tuple           =   intermediate_size
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   hidden_act
											SCREAMING_SNAKE_CASE__						:       Any           =   hidden_dropout_prob
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE__						:       int           =   initializer_range
											SCREAMING_SNAKE_CASE__						:       int           =   layer_norm_eps
											SCREAMING_SNAKE_CASE__						:       Dict           =   patch_size
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   qkv_bias
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   frequency_stride
											SCREAMING_SNAKE_CASE__						:       Any           =   time_stride
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   max_length
											SCREAMING_SNAKE_CASE__						:       Any           =   num_mel_bins




