

import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
					from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
					lowerCAmelCase_       	=	True
					from torch.cuda.amp import autocast


lowerCAmelCase_       	=	logging.getLogger(__name__)





@dataclass
class 	snake_case_   :







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   str									=				field(
			    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}					)
			SCREAMING_SNAKE_CASE     :   Optional[str]									=				field(
			    default=__A	,       metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}	,       )
			SCREAMING_SNAKE_CASE     :   Optional[bool]									=				field(
			    default=__A	,       metadata={"help": "Whether to freeze the feature extractor layers of the model."}					)
			SCREAMING_SNAKE_CASE     :   Optional[bool]									=				field(
			    default=__A	,       metadata={"help": "Whether to log verbose messages or not."}	,       )
			SCREAMING_SNAKE_CASE     :   Optional[float]									=				field(
			    default=2.0	,       metadata={"help": "Maximum temperature for gumbel softmax."}					)
			SCREAMING_SNAKE_CASE     :   Optional[float]									=				field(
			    default=0.5	,       metadata={"help": "Minimum temperature for gumbel softmax."}					)
			SCREAMING_SNAKE_CASE     :   Optional[float]									=				field(
			    default=0.99_9995	,       metadata={"help": "Decay of gumbel temperature during training."}					)






def   __SCREAMING_SNAKE_CASE					(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       ):
						logging.basicConfig(
						    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''  ,		datefmt='''%m/%d/%Y %H:%M:%S'''  ,		handlers=[logging.StreamHandler(sys.stdout       )]  ,		)
						snake_case_   =	logging.WARNING
						if model_args.verbose_logging:
												snake_case_   =	logging.DEBUG
						elif trainer_utils.is_main_process(training_args.local_rank       ):
												snake_case_   =	logging.INFO
						logger.setLevel(SCREAMING_SNAKE_CASE__       )





@dataclass
class 	snake_case_   :







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   str									=				field(
			    default=__A	,       metadata={"help": "The name of the dataset to use (via the datasets library)."}					)
			SCREAMING_SNAKE_CASE     :   Optional[str]									=				field(
			    default=__A	,       metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}					)
			SCREAMING_SNAKE_CASE     :   Optional[str]									=				field(
			    default="train"	,       metadata={
			        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
			    }	,       )
			SCREAMING_SNAKE_CASE     :   Optional[str]									=				field(
			    default="validation"	,       metadata={
			        "help": (
			            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
			        )
			    }	,       )
			SCREAMING_SNAKE_CASE     :   Optional[str]									=				field(
			    default="file"	,       metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}	,       )
			SCREAMING_SNAKE_CASE     :   bool									=				field(
			    default=__A	,       metadata={"help": "Overwrite the cached preprocessed datasets or not."}					)
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				field(
			    default=1	,       metadata={
			        "help": "The percentage of the train set used as validation set in case there's no validation split"
			    }	,       )
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				field(
			    default=__A	,       metadata={"help": "The number of processes to use for the preprocessing."}	,       )
			SCREAMING_SNAKE_CASE     :   Optional[float]									=				field(
			    default=20.0	,       metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}					)






@dataclass
class 	snake_case_   :







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   WavaVecaForPreTraining
			SCREAMING_SNAKE_CASE     :   WavaVecaFeatureExtractor
			SCREAMING_SNAKE_CASE     :   Union[bool, str]									=				"longest"
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				None
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				None







			def __call__(     self :				Dict							,   _UpperCamelCase :				List[Dict[str, Union[List[int], torch.Tensor]]]							)      ->Dict[str, torch.Tensor]:
									# reformat list to dict and set to pytorch format
									snake_case_   =	self.feature_extractor.pad(
									    _UpperCamelCase							,   max_length=self.max_length							,   padding=self.padding							,   pad_to_multiple_of=self.pad_to_multiple_of							,   return_tensors='''pt'''							,   )
									snake_case_   =	self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]							)

									snake_case_   =	batch['''input_values'''].shape[0]

									# make sure that no loss is computed on padded inputs
									if batch["attention_mask"] is not None:
															# compute real output lengths according to convolution formula
															snake_case_   =	self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1							)							).to(
															    torch.long							)

															snake_case_   =	torch.zeros(
															    (batch_size, mask_indices_seq_length)							,   dtype=torch.long							,   device=batch['''input_values'''].device							)

															# these two operations makes sure that all values
															# before the output lengths indices are attended to
															snake_case_   =	1
															snake_case_   =	attention_mask.flip([-1]							).cumsum(-1							).flip([-1]							).bool()

									# sample randomly masked indices
									snake_case_   =	_compute_mask_indices(
									    (batch_size, mask_indices_seq_length)							,   self.model.config.mask_time_prob							,   self.model.config.mask_time_length							,   attention_mask=_UpperCamelCase							,   min_masks=2							,   )

									return batch






class 	snake_case_   (		__A					):







			'''simple docstring'''

			def __init__(     self :				Optional[int]							,   *_UpperCamelCase :				Tuple							,   _UpperCamelCase :				Optional[Any]=1							,   _UpperCamelCase :				str=0							,   _UpperCamelCase :				List[str]=1.0							,   **_UpperCamelCase :				Union[str, Any]							)      ->str:
									super().__init__(*_UpperCamelCase							,   **_UpperCamelCase							)
									snake_case_   =	0
									snake_case_   =	max_gumbel_temp
									snake_case_   =	min_gumbel_temp
									snake_case_   =	gumbel_temp_decay







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				nn.Module							,   _UpperCamelCase :				Dict[str, Union[torch.Tensor, Any]]							)      ->torch.Tensor:
									model.train()
									snake_case_   =	self._prepare_inputs(_UpperCamelCase							)

									if self.use_amp:
															with autocast():
																					snake_case_   =	self.compute_loss(_UpperCamelCase							,   _UpperCamelCase							)
									else:
															snake_case_   =	self.compute_loss(_UpperCamelCase							,   _UpperCamelCase							)

									if self.args.n_gpu > 1 or self.deepspeed:
															if model.module.config.ctc_loss_reduction == "mean":
																					snake_case_   =	loss.mean()
															elif model.module.config.ctc_loss_reduction == "sum":
																					snake_case_   =	loss.sum() / (inputs['''mask_time_indices''']).sum()
															else:
																					raise ValueError(f'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''							)

									if self.args.gradient_accumulation_steps > 1:
															snake_case_   =	loss / self.args.gradient_accumulation_steps

									if self.use_amp:
															self.scaler.scale(_UpperCamelCase							).backward()
									elif self.use_apex:
															with amp.scale_loss(_UpperCamelCase							,   self.optimizer							) as scaled_loss:
																					scaled_loss.backward()
									elif self.deepspeed:
															self.deepspeed.backward(_UpperCamelCase							)
									else:
															loss.backward()

									self.num_update_step += 1
									# make sure gumbel softmax temperature is decayed
									if self.args.n_gpu > 1 or self.deepspeed:
															model.module.set_gumbel_temperature(
															    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step							,   self.min_gumbel_temp							)							)
									else:
															model.set_gumbel_temperature(
															    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step							,   self.min_gumbel_temp							)							)

									return loss.detach()






def   __SCREAMING_SNAKE_CASE					():
						# See all possible arguments in src/transformers/training_args.py
						# or by passing the --help flag to this script.
						# We now keep distinct sets of args, for a cleaner separation of concerns.

						snake_case_   =	HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

						snake_case_,    snake_case_,    snake_case_   =	parser.parse_args_into_dataclasses()
						configure_logger(SCREAMING_SNAKE_CASE__  ,		SCREAMING_SNAKE_CASE__       )

						# Downloading and loading a dataset from the hub.
						snake_case_   =	load_dataset(data_args.dataset_name  ,		data_args.dataset_config_name  ,		cache_dir=model_args.cache_dir       )

						if "validation" not in datasets.keys():
												# make sure only "validation" and "train" keys remain"
												snake_case_   =	DatasetDict()
												snake_case_   =	load_dataset(
												    data_args.dataset_name  ,		data_args.dataset_config_name  ,		split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''  ,		cache_dir=model_args.cache_dir  ,		)
												snake_case_   =	load_dataset(
												    data_args.dataset_name  ,		data_args.dataset_config_name  ,		split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''  ,		cache_dir=model_args.cache_dir  ,		)
						else:
												# make sure only "validation" and "train" keys remain"
												snake_case_   =	DatasetDict()
												snake_case_   =	load_dataset(
												    data_args.dataset_name  ,		data_args.dataset_config_name  ,		split='''validation'''  ,		cache_dir=model_args.cache_dir  ,		)
												snake_case_   =	load_dataset(
												    data_args.dataset_name  ,		data_args.dataset_config_name  ,		split=F'''{data_args.train_split_name}'''  ,		cache_dir=model_args.cache_dir  ,		)

						# only normalized-inputs-training is supported
						snake_case_   =	WavaVecaFeatureExtractor.from_pretrained(
						    model_args.model_name_or_path  ,		cache_dir=model_args.cache_dir  ,		do_normalize=SCREAMING_SNAKE_CASE__       )

						def prepare_dataset(SCREAMING_SNAKE_CASE__       ):
												# check that all files have the correct sampling rate
												snake_case_,    snake_case_   =	librosa.load(batch[data_args.speech_file_column]  ,		sr=feature_extractor.sampling_rate       )
												return batch

						# load audio files into numpy arrays
						snake_case_   =	datasets.map(
						    SCREAMING_SNAKE_CASE__  ,		num_proc=data_args.preprocessing_num_workers  ,		remove_columns=datasets['''train'''].column_names       )

						# filter audio files that are too long
						snake_case_   =	vectorized_datasets.filter(
						    lambda SCREAMING_SNAKE_CASE__       : len(data['''speech''']       ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate       )       )

						def normalize(SCREAMING_SNAKE_CASE__       ):
												return feature_extractor(batch['''speech''']  ,		sampling_rate=feature_extractor.sampling_rate       )

						# normalize and transform to `BatchFeatures`
						snake_case_   =	vectorized_datasets.map(
						    SCREAMING_SNAKE_CASE__  ,		batched=SCREAMING_SNAKE_CASE__  ,		num_proc=data_args.preprocessing_num_workers  ,		load_from_cache_file=not data_args.overwrite_cache  ,		remove_columns=vectorized_datasets['''train'''].column_names  ,		)

						# pretraining is only supported for "newer" stable layer norm architecture
						# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
						snake_case_   =	WavaVecaConfig.from_pretrained(
						    model_args.model_name_or_path  ,		cache_dir=model_args.cache_dir  ,		gradient_checkpointing=training_args.gradient_checkpointing  ,		)

						if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
												raise ValueError(
												    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
												    ''' ``config.feat_extract_norm=\'layer\''''       )

						snake_case_   =	WavaVecaForPreTraining(SCREAMING_SNAKE_CASE__       )

						snake_case_   =	DataCollatorForWavaVecaPretraining(model=SCREAMING_SNAKE_CASE__  ,		feature_extractor=SCREAMING_SNAKE_CASE__       )

						snake_case_   =	WavaVecaPreTrainer(
						    model=SCREAMING_SNAKE_CASE__  ,		data_collator=SCREAMING_SNAKE_CASE__  ,		args=SCREAMING_SNAKE_CASE__  ,		train_dataset=vectorized_datasets['''train''']  ,		eval_dataset=vectorized_datasets['''validation''']  ,		tokenizer=SCREAMING_SNAKE_CASE__  ,		max_gumbel_temp=model_args.max_gumbel_temperature  ,		min_gumbel_temp=model_args.min_gumbel_temperature  ,		gumbel_temp_decay=model_args.gumbel_temperature_decay  ,		)
						trainer.train()


if __name__ == "__main__":
					main()


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_       	=	logging.get_logger(__name__)

lowerCAmelCase_       	=	{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    }
}

lowerCAmelCase_       	=	{
    '''google/bigbird-roberta-base''': 40_96,
    '''google/bigbird-roberta-large''': 40_96,
    '''google/bigbird-base-trivia-itc''': 40_96,
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   List[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   List[Any]									=				["input_ids", "attention_mask"]
			SCREAMING_SNAKE_CASE     :   List[int]									=				[]







			def __init__(     self :				List[str]							,   _UpperCamelCase :				List[str]							,   _UpperCamelCase :				Dict="<unk>"							,   _UpperCamelCase :				List[str]="<s>"							,   _UpperCamelCase :				Tuple="</s>"							,   _UpperCamelCase :				Any="<pad>"							,   _UpperCamelCase :				Any="[SEP]"							,   _UpperCamelCase :				Optional[Any]="[MASK]"							,   _UpperCamelCase :				Any="[CLS]"							,   _UpperCamelCase :				Optional[Dict[str, Any]] = None							,   **_UpperCamelCase :				Dict							,   )      ->None:
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else bos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else eos_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else unk_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else pad_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else cls_token
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else sep_token

									# Mask token behave like a normal word, i.e. include the space before it
									snake_case_   =	AddedToken(_UpperCamelCase							,   lstrip=_UpperCamelCase							,   rstrip=_UpperCamelCase							) if isinstance(_UpperCamelCase							,   _UpperCamelCase							) else mask_token

									snake_case_   =	{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=_UpperCamelCase							,   eos_token=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   sp_model_kwargs=self.sp_model_kwargs							,   **_UpperCamelCase							,   )

									snake_case_   =	vocab_file

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(_UpperCamelCase							)







			@property
			def 				snake_case__(     self :				str							)      ->List[Any]:
									return self.sp_model.get_piece_size()







			def 				snake_case__(     self :				int							)      ->Union[str, Any]:
									snake_case_   =	{self.convert_ids_to_tokens(_UpperCamelCase							): i for i in range(self.vocab_size							)}
									vocab.update(self.added_tokens_encoder							)
									return vocab







			def __getstate__(     self :				Tuple							)      ->Any:
									snake_case_   =	self.__dict__.copy()
									snake_case_   =	None
									return state







			def __setstate__(     self :				str							,   _UpperCamelCase :				List[Any]							)      ->List[str]:
									snake_case_   =	d

									# for backward compatibility
									if not hasattr(self							,   '''sp_model_kwargs'''							):
															snake_case_   =	{}

									snake_case_   =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(self.vocab_file							)







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				str							)      ->List[str]:
									return self.sp_model.encode(_UpperCamelCase							,   out_type=_UpperCamelCase							)







			def 				snake_case__(     self :				str							,   _UpperCamelCase :				List[str]							)      ->Tuple:
									return self.sp_model.piece_to_id(_UpperCamelCase							)







			def 				snake_case__(     self :				Union[str, Any]							,   _UpperCamelCase :				str							)      ->List[Any]:
									snake_case_   =	self.sp_model.IdToPiece(_UpperCamelCase							)
									return token







			def 				snake_case__(     self :				Dict							,   _UpperCamelCase :				Optional[int]							)      ->List[str]:
									snake_case_   =	[]
									snake_case_   =	''''''
									snake_case_   =	False
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					if not prev_is_special:
																											out_string += " "
																					out_string += self.sp_model.decode(_UpperCamelCase							) + token
																					snake_case_   =	True
																					snake_case_   =	[]
															else:
																					current_sub_tokens.append(_UpperCamelCase							)
																					snake_case_   =	False
									out_string += self.sp_model.decode(_UpperCamelCase							)
									return out_string.strip()







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				bool = False							,   _UpperCamelCase :				bool = None							,   _UpperCamelCase :				bool = True							,   **_UpperCamelCase :				List[str]							,   )      ->str:
									snake_case_   =	kwargs.pop('''use_source_tokenizer'''							,   _UpperCamelCase							)

									snake_case_   =	self.convert_ids_to_tokens(_UpperCamelCase							,   skip_special_tokens=_UpperCamelCase							)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									snake_case_   =	[]
									snake_case_   =	[]
									for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																					continue
															if token in self.added_tokens_encoder:
																					if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)
																											snake_case_   =	[]
																					sub_texts.append(_UpperCamelCase							)
															else:
																					current_sub_text.append(_UpperCamelCase							)
									if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase							)							)

									# Mimic the behavior of the Rust tokenizer:
									# No space before [MASK] and [SEP]
									if spaces_between_special_tokens:
															snake_case_   =	re.sub(R''' (\[(MASK|SEP)\])'''							,   R'''\1'''							,   ''' '''.join(_UpperCamelCase							)							)
									else:
															snake_case_   =	''''''.join(_UpperCamelCase							)

									snake_case_   =	(
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
															snake_case_   =	self.clean_up_tokenization(_UpperCamelCase							)
															return clean_text
									else:
															return text







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									if not os.path.isdir(_UpperCamelCase							):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
															return
									snake_case_   =	os.path.join(
									    _UpperCamelCase							,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

									if os.path.abspath(self.vocab_file							) != os.path.abspath(_UpperCamelCase							) and os.path.isfile(self.vocab_file							):
															copyfile(self.vocab_file							,   _UpperCamelCase							)
									elif not os.path.isfile(self.vocab_file							):
															with open(_UpperCamelCase							,   '''wb'''							) as fi:
																					snake_case_   =	self.sp_model.serialized_model_proto()
																					fi.write(_UpperCamelCase							)

									return (out_vocab_file,)







			def 				snake_case__(     self :				Tuple							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									snake_case_   =	[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep







			def 				snake_case__(     self :				List[str]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							,   _UpperCamelCase :				bool = False							)      ->List[int]:
									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_UpperCamelCase							,   token_ids_a=_UpperCamelCase							,   already_has_special_tokens=_UpperCamelCase							)

									if token_ids_a is None:
															return [1] + ([0] * len(_UpperCamelCase							)) + [1]
									return [1] + ([0] * len(_UpperCamelCase							)) + [1] + ([0] * len(_UpperCamelCase							)) + [1]







			def 				snake_case__(     self :				List[Any]							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]