





"""simple docstring"""




import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCamelCase_	 =16
UpperCamelCase_	 =32





def 			a_							(				_lowercase  ):
						return int(x / 2**20  )




class 					_a		:






				def __enter__(					self						:       Optional[Any]				)      -> Tuple:




										'''simple docstring'''



										gc.collect()
										torch.cuda.empty_cache()
										torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
										_UpperCamelCase     :   str      =       torch.cuda.memory_allocated()
										_UpperCamelCase     :   str      =       psutil.Process()

										_UpperCamelCase     :   str      =       self.cpu_mem_used()
										_UpperCamelCase     :   str      =       True
										_UpperCamelCase     :   Tuple      =       threading.Thread(target=self.peak_monitor_func				)
										_UpperCamelCase     :   List[str]      =       True
										peak_monitor_thread.start()
										return self






				def 							snake_case      (					self						:       Union[str, Any]				)      -> Any:




										'''simple docstring'''



										return self.process.memory_info().rss






				def 							snake_case      (					self						:       int				)      -> Optional[int]:




										'''simple docstring'''



										_UpperCamelCase     :   List[str]      =       -1

										while True:
																_UpperCamelCase     :   int      =       max(self.cpu_mem_used(),					self.cpu_peak				)

																# can't sleep or will not catch the peak right (this comment is here on purpose)
																# time.sleep(0.001) # 1msec

																if not self.peak_monitoring:
																						break






				def __exit__(					self						:       int,					*lowerCAmelCase__						:       List[Any]				)      -> Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase     :   Optional[Any]      =       False

										gc.collect()
										torch.cuda.empty_cache()
										_UpperCamelCase     :   int      =       torch.cuda.memory_allocated()
										_UpperCamelCase     :   str      =       torch.cuda.max_memory_allocated()
										_UpperCamelCase     :   Dict      =       bamb(self.end - self.begin				)
										_UpperCamelCase     :   Any      =       bamb(self.peak - self.begin				)

										_UpperCamelCase     :   int      =       self.cpu_mem_used()
										_UpperCamelCase     :   str      =       bamb(self.cpu_end - self.cpu_begin				)
										_UpperCamelCase     :   List[Any]      =       bamb(self.cpu_peak - self.cpu_begin				)
										# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
						from accelerate.test_utils.training import mocked_dataloaders

						UpperCamelCase_	 =mocked_dataloaders  # noqa: F811





def 			a_							(				_lowercase			,		_lowercase  ):

						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''			,		__UpperCAmelCase  ) == "1":
												_UpperCamelCase     :   Tuple      =       2

						# New Code #
						# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
						_UpperCamelCase     :   Union[str, Any]      =       FullyShardedDataParallelPlugin(
						    state_dict_config=FullStateDictConfig(offload_to_cpu=__UpperCAmelCase			,		ranka_only=__UpperCAmelCase  )			,		optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=__UpperCAmelCase			,		ranka_only=__UpperCAmelCase  )			,		)

						# Initialize accelerator
						if args.with_tracking:
												_UpperCamelCase     :   int      =       Accelerator(
												    cpu=args.cpu			,		mixed_precision=args.mixed_precision			,		log_with='''wandb'''			,		project_dir=args.logging_dir			,		fsdp_plugin=__UpperCAmelCase			,		)
						else:
												_UpperCamelCase     :   Union[str, Any]      =       Accelerator(fsdp_plugin=__UpperCAmelCase  )
						accelerator.print(accelerator.distributed_type  )

						if hasattr(args.checkpointing_steps			,		'''isdigit'''  ):
												if args.checkpointing_steps == "epoch":
																		_UpperCamelCase     :   List[Any]      =       args.checkpointing_steps
												elif args.checkpointing_steps.isdigit():
																		_UpperCamelCase     :   str      =       int(args.checkpointing_steps  )
												else:
																		raise ValueError(
																		    F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""  )
						else:
												_UpperCamelCase     :   str      =       None
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						_UpperCamelCase     :   List[str]      =       config['''lr''']
						_UpperCamelCase     :   Optional[int]      =       int(config['''num_epochs''']  )
						_UpperCamelCase     :   Union[str, Any]      =       int(config['''seed''']  )
						_UpperCamelCase     :   int      =       int(config['''batch_size''']  )

						# We need to initialize the trackers we use, and also store our configuration
						if args.with_tracking:
												_UpperCamelCase     :   int      =       vars(__UpperCAmelCase  )
												accelerator.init_trackers('''fsdp_glue_no_trainer'''			,		__UpperCAmelCase  )

						_UpperCamelCase     :   str      =       AutoTokenizer.from_pretrained(args.model_name_or_path  )
						_UpperCamelCase     :   Dict      =       load_dataset('''glue'''			,		'''mrpc'''  )
						_UpperCamelCase     :   Optional[Any]      =       evaluate.load('''glue'''			,		'''mrpc'''  )

						def tokenize_function(_lowercase  ):
												# max_length=None => use the model max length (it's actually the default)
												_UpperCamelCase     :   int      =       tokenizer(examples['''sentence1''']			,		examples['''sentence2''']			,		truncation=__UpperCAmelCase			,		max_length=__UpperCAmelCase  )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												_UpperCamelCase     :   List[Any]      =       datasets.map(
												    __UpperCAmelCase			,		batched=__UpperCAmelCase			,		remove_columns=['''idx''', '''sentence1''', '''sentence2''']			,		)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						_UpperCamelCase     :   Any      =       tokenized_datasets.rename_column('''label'''			,		'''labels'''  )

						# If the batch size is too big we use gradient accumulation
						_UpperCamelCase     :   int      =       1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												_UpperCamelCase     :   Optional[int]      =       batch_size // MAX_GPU_BATCH_SIZE
												_UpperCamelCase     :   Optional[Any]      =       MAX_GPU_BATCH_SIZE

						def collate_fn(_lowercase  ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												_UpperCamelCase     :   Optional[Any]      =       128 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		_UpperCamelCase     :   List[str]      =       16
												elif accelerator.mixed_precision != "no":
																		_UpperCamelCase     :   Union[str, Any]      =       8
												else:
																		_UpperCamelCase     :   List[str]      =       None

												return tokenizer.pad(
												    __UpperCAmelCase			,		padding='''longest'''			,		max_length=__UpperCAmelCase			,		pad_to_multiple_of=__UpperCAmelCase			,		return_tensors='''pt'''			,		)

						# Instantiate dataloaders.
						_UpperCamelCase     :   Dict      =       DataLoader(
						    tokenized_datasets['''train''']			,		shuffle=__UpperCAmelCase			,		collate_fn=__UpperCAmelCase			,		batch_size=__UpperCAmelCase  )
						_UpperCamelCase     :   Optional[Any]      =       DataLoader(
						    tokenized_datasets['''validation''']			,		shuffle=__UpperCAmelCase			,		collate_fn=__UpperCAmelCase			,		batch_size=__UpperCAmelCase  )

						set_seed(__UpperCAmelCase  )

						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						_UpperCamelCase     :   int      =       AutoModelForSequenceClassification.from_pretrained(
						    args.model_name_or_path			,		return_dict=__UpperCAmelCase			,		low_cpu_mem_usage=__UpperCAmelCase  )

						# New Code #
						# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
						_UpperCamelCase     :   Any      =       accelerator.prepare(__UpperCAmelCase  )
						accelerator.print(__UpperCAmelCase  )

						# Instantiate optimizer
						# New Code #
						# For FSDP feature, at present it doesn't support multiple parameter groups,
						# so we need to create a single parameter group for the whole model
						_UpperCamelCase     :   Union[str, Any]      =       torch.optim.AdamW(params=model.parameters()			,		lr=__UpperCAmelCase			,		weight_decay=2E-4  )

						# Instantiate scheduler
						_UpperCamelCase     :   Optional[int]      =       get_linear_schedule_with_warmup(
						    optimizer=__UpperCAmelCase			,		num_warmup_steps=10			,		num_training_steps=(len(__UpperCAmelCase  ) * num_epochs) // gradient_accumulation_steps			,		)

						# New Code #
						# For FSDP feature, prepare everything except the model as we have already prepared the model
						# before creating the optimizer
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase  ,				_UpperCamelCase     :   List[Any]      =       accelerator.prepare(
						    __UpperCAmelCase			,		__UpperCAmelCase			,		__UpperCAmelCase			,		__UpperCAmelCase  )

						_UpperCamelCase     :   List[Any]      =       0

						# Potentially load in the weights and states from a previous save
						if args.resume_from_checkpoint:
												if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
																		accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""  )
																		accelerator.load_state(args.resume_from_checkpoint  )
																		_UpperCamelCase     :   Union[str, Any]      =       os.path.basename(args.resume_from_checkpoint  )
												else:
																		# Get the most recent checkpoint
																		_UpperCamelCase     :   Dict      =       [f.name for f in os.scandir(os.getcwd()  ) if f.is_dir()]
																		dirs.sort(key=os.path.getctime  )
																		_UpperCamelCase     :   Optional[int]      =       dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
												# Extract `epoch_{i}` or `step_{i}`
												_UpperCamelCase     :   Dict      =       os.path.splitext(__UpperCAmelCase  )[0]

												if "epoch" in training_difference:
																		num_epochs -= int(training_difference.replace('''epoch_'''			,		''''''  )  )
																		_UpperCamelCase     :   Tuple      =       None
												else:
																		_UpperCamelCase     :   Tuple      =       int(training_difference.replace('''step_'''			,		''''''  )  )
																		num_epochs -= resume_step // len(__UpperCAmelCase  )
																		# If resuming by step, we also need to know exactly how far into the DataLoader we went
																		_UpperCamelCase     :   Tuple      =       (num_epochs * len(__UpperCAmelCase  )) - resume_step

    # Now we train the model
						for epoch in range(__UpperCAmelCase  ):
												# New Code #
												# context manager to track the peak memory usage during the training epoch
												with TorchTracemalloc() as tracemalloc:
																		model.train()
																		if args.with_tracking:
																								_UpperCamelCase     :   int      =       0
																		for step, batch in enumerate(__UpperCAmelCase  ):
																								# We need to skip steps until we reach the resumed step
																								if args.resume_from_checkpoint and epoch == 0:
																														if resume_step is not None and step < resume_step:
																																				pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device  )
																								_UpperCamelCase     :   Union[str, Any]      =       model(**__UpperCAmelCase  )
																								_UpperCamelCase     :   Any      =       outputs.loss
																								_UpperCamelCase     :   Union[str, Any]      =       loss / gradient_accumulation_steps
																								# We keep track of the loss at each epoch
																								if args.with_tracking:
																														total_loss += loss.detach().float()
																								accelerator.backward(__UpperCAmelCase  )
																								if step % gradient_accumulation_steps == 0:
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()
																														# accelerator.print(lr_scheduler.get_lr())

																								overall_step += 1

																								if isinstance(__UpperCAmelCase			,		__UpperCAmelCase  ):
																														_UpperCamelCase     :   Optional[int]      =       F"""step_{overall_step}"""
																														if overall_step % checkpointing_steps == 0:
																																				if args.output_dir is not None:
																																										_UpperCamelCase     :   Dict      =       os.path.join(args.output_dir			,		__UpperCAmelCase  )
																																				accelerator.save_state(__UpperCAmelCase  )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin  )  )  )
												accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used  )  )
												accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked  )  )
												accelerator.print(
												    '''Total Peak Memory consumed during the train (max): {}'''.format(
												        tracemalloc.peaked + bamb(tracemalloc.begin  )  )  )
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        '''train_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin  ),
																		    }			,		step=__UpperCAmelCase			,		)

												# New Code #
												# context manager to track the peak memory usage during the evaluation
												with TorchTracemalloc() as tracemalloc:
																		model.eval()
																		for step, batch in enumerate(__UpperCAmelCase  ):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device  )
																								with torch.no_grad():
																														_UpperCamelCase     :   List[Any]      =       model(**__UpperCAmelCase  )
																								_UpperCamelCase     :   int      =       outputs.logits.argmax(dim=-1  )
																								_UpperCamelCase  ,				_UpperCamelCase     :   Any      =       accelerator.gather_for_metrics((predictions, batch['''labels'''])  )
																								metric.add_batch(
																								    predictions=__UpperCAmelCase			,		references=__UpperCAmelCase			,		)

																		_UpperCamelCase     :   Any      =       metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(F"""epoch {epoch}:"""			,		__UpperCAmelCase  )
																		if args.with_tracking:
																								accelerator.log(
																								    {
																								        '''accuracy''': eval_metric['''accuracy'''],
																								        '''f1''': eval_metric['''f1'''],
																								        '''train_loss''': total_loss.item() / len(__UpperCAmelCase  ),
																								    }			,		step=__UpperCAmelCase			,		)

																		if checkpointing_steps == "epoch":
																								_UpperCamelCase     :   List[str]      =       F"""epoch_{epoch}"""
																								if args.output_dir is not None:
																														_UpperCamelCase     :   Tuple      =       os.path.join(args.output_dir			,		__UpperCAmelCase  )
																								accelerator.save_state(__UpperCAmelCase  )
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
												accelerator.print('''Memory before entering the eval : {}'''.format(bamb(tracemalloc.begin  )  )  )
												accelerator.print('''Memory consumed at the end of the eval (end-begin): {}'''.format(tracemalloc.used  )  )
												accelerator.print('''Peak Memory consumed during the eval (max-begin): {}'''.format(tracemalloc.peaked  )  )
												accelerator.print(
												    '''Total Peak Memory consumed during the eval (max): {}'''.format(tracemalloc.peaked + bamb(tracemalloc.begin  )  )  )
												# Logging the peak memory usage of the GPU to the tracker
												if args.with_tracking:
																		accelerator.log(
																		    {
																		        '''eval_total_peak_memory''': tracemalloc.peaked + bamb(tracemalloc.begin  ),
																		    }			,		step=__UpperCAmelCase			,		)

						if args.with_tracking:
												accelerator.end_training()





def 			a_							(				):
						_UpperCamelCase     :   List[Any]      =       argparse.ArgumentParser(description='''Simple example of training script.'''  )
						parser.add_argument(
						    '''--mixed_precision'''			,		type=__UpperCAmelCase			,		default=__UpperCAmelCase			,		choices=['''no''', '''fp16''', '''bf16''', '''fp8''']			,		help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''			,		)
						parser.add_argument('''--cpu'''			,		action='''store_true'''			,		help='''If passed, will train on the CPU.'''  )
						parser.add_argument(
						    '''--checkpointing_steps'''			,		type=__UpperCAmelCase			,		default=__UpperCAmelCase			,		help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''			,		)
						parser.add_argument(
						    '''--resume_from_checkpoint'''			,		type=__UpperCAmelCase			,		default=__UpperCAmelCase			,		help='''If the training should continue from a checkpoint folder.'''			,		)
						parser.add_argument(
						    '''--with_tracking'''			,		action='''store_true'''			,		help='''Whether to load in all available experiment trackers from the environment and use them for logging.'''			,		)
						parser.add_argument(
						    '''--output_dir'''			,		type=__UpperCAmelCase			,		default='''.'''			,		help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''			,		)
						parser.add_argument(
						    '''--logging_dir'''			,		type=__UpperCAmelCase			,		default='''logs'''			,		help='''Location on where to store experiment tracking logs`'''			,		)
						parser.add_argument(
						    '''--model_name_or_path'''			,		type=__UpperCAmelCase			,		help='''Path to pretrained model or model identifier from huggingface.co/models.'''			,		required=__UpperCAmelCase			,		)
						_UpperCamelCase     :   List[str]      =       parser.parse_args()
						_UpperCamelCase     :   Optional[Any]      =       {'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
						training_function(__UpperCAmelCase			,		__UpperCAmelCase  )


if __name__ == "__main__":
						main()




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case__						      =			logging.get_logger(__name__)


snake_case__						      =			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all MVP models at https://huggingface.co/models?filter=mvp
snake_case__						      =			{
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
}

snake_case__						      =			{
    'RUCAIBox/mvp': 1024,
}
@lru_cache()
def    __magic_name__(	)    ->     List[str]:



      '''simple docstring'''

      _lowerCamelCase           =					(
          list(range(ord('''!'''		)			,    ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)			,    ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)			,    ord('''ÿ'''		) + 1		)		)
      )
      _lowerCamelCase           =					bs[:]
      _lowerCamelCase           =					0
      for b in range(2**8		):
            if b not in bs:
                  bs.append(__UpperCAmelCase		)
                  cs.append(2**8 + n		)
                  n += 1
      _lowerCamelCase           =					[chr(__UpperCAmelCase		) for n in cs]
      return dict(zip(__UpperCAmelCase			,    __UpperCAmelCase		)		)
def    __magic_name__(	__UpperCAmelCase		)    ->     List[Any]:



      '''simple docstring'''

      _lowerCamelCase           =					set()
      _lowerCamelCase           =					word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)		)
            _lowerCamelCase           =					char
      return pairs




class 			UpperCamelCase					(			__lowercase			):

     '''simple docstring'''

     A_					    =	VOCAB_FILES_NAMES
     A_					    =	PRETRAINED_VOCAB_FILES_MAP
     A_					    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_					    =	['input_ids', 'attention_mask']



     def __init__(       self					,					A_					,					A_					,					A_="replace"					,					A_="<s>"					,					A_="</s>"					,					A_="</s>"					,					A_="<s>"					,					A_="<unk>"					,					A_="<pad>"					,					A_="<mask>"					,					A_=False					,					**A_					,					)			->					Union[str, Any]:






           """simple docstring"""
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else bos_token
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else eos_token
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else sep_token
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else cls_token
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else unk_token
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           _lowerCamelCase           =					AddedToken(A_					,					lstrip=A_					,					rstrip=A_    ) if isinstance(A_					,					A_    ) else mask_token

           super().__init__(
               errors=A_					,					bos_token=A_					,					eos_token=A_					,					unk_token=A_					,					sep_token=A_					,					cls_token=A_					,					pad_token=A_					,					mask_token=A_					,					add_prefix_space=A_					,					**A_					,					)
           with open(A_					,					encoding='''utf-8'''    ) as vocab_handle:
                 _lowerCamelCase           =					json.load(A_    )
           _lowerCamelCase           =					{v: k for k, v in self.encoder.items()}
           _lowerCamelCase           =					errors  # how to handle errors in decoding
           _lowerCamelCase           =					bytes_to_unicode()
           _lowerCamelCase           =					{v: k for k, v in self.byte_encoder.items()}
           with open(A_					,					encoding='''utf-8'''    ) as merges_handle:
                 _lowerCamelCase           =					merges_handle.read().split('''\n'''    )[1:-1]
           _lowerCamelCase           =					[tuple(merge.split()    ) for merge in bpe_merges]
           _lowerCamelCase           =					dict(zip(A_					,					range(len(A_    )    )    )    )
           _lowerCamelCase           =					{}
           _lowerCamelCase           =					add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           _lowerCamelCase           =					re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )



     @property
     def 		UpperCamelCase_    (       self    )			->					Dict:






           """simple docstring"""
           return len(self.encoder    )



     def 		UpperCamelCase_    (       self    )			->					Tuple:






           """simple docstring"""
           return dict(self.encoder					,					**self.added_tokens_encoder    )



     def 		UpperCamelCase_    (       self					,					A_    )			->					str:






           """simple docstring"""
           if token in self.cache:
                 return self.cache[token]
           _lowerCamelCase           =					tuple(A_    )
           _lowerCamelCase           =					get_pairs(A_    )

           if not pairs:
                 return token

           while True:
                 _lowerCamelCase           =					min(A_					,					key=lambda A_    : self.bpe_ranks.get(A_					,					float('''inf'''    )    )    )
                 if bigram not in self.bpe_ranks:
                       break
                 _lowerCamelCase     ,       _lowerCamelCase           =					bigram
                 _lowerCamelCase           =					[]
                 _lowerCamelCase           =					0
                 while i < len(A_    ):
                       try:
                             _lowerCamelCase           =					word.index(A_					,					A_    )
                       except ValueError:
                             new_word.extend(word[i:]    )
                             break
                       else:
                             new_word.extend(word[i:j]    )
                             _lowerCamelCase           =					j

                       if word[i] == first and i < len(A_    ) - 1 and word[i + 1] == second:
                             new_word.append(first + second    )
                             i += 2
                       else:
                             new_word.append(word[i]    )
                             i += 1
                 _lowerCamelCase           =					tuple(A_    )
                 _lowerCamelCase           =					new_word
                 if len(A_    ) == 1:
                       break
                 else:
                       _lowerCamelCase           =					get_pairs(A_    )
           _lowerCamelCase           =					''' '''.join(A_    )
           _lowerCamelCase           =					word
           return word



     def 		UpperCamelCase_    (       self					,					A_    )			->					List[Any]:






           """simple docstring"""
           _lowerCamelCase           =					[]
           for token in re.findall(self.pat					,					A_    ):
                 _lowerCamelCase           =					''''''.join(
                     self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_    ).split(''' '''    )    )
           return bpe_tokens



     def 		UpperCamelCase_    (       self					,					A_    )			->					Optional[Any]:






           """simple docstring"""
           return self.encoder.get(A_					,					self.encoder.get(self.unk_token    )    )



     def 		UpperCamelCase_    (       self					,					A_    )			->					Any:






           """simple docstring"""
           return self.decoder.get(A_    )



     def 		UpperCamelCase_    (       self					,					A_    )			->					Dict:






           """simple docstring"""
           _lowerCamelCase           =					''''''.join(A_    )
           _lowerCamelCase           =					bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''					,					errors=self.errors    )
           return text



     def 		UpperCamelCase_    (       self					,					A_					,					A_ = None    )			->					Tuple[str]:






           """simple docstring"""
           if not os.path.isdir(A_    ):
                 logger.error(F'Vocabulary path ({save_directory}) should be a directory'    )
                 return
           _lowerCamelCase           =					os.path.join(
               A_					,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
           _lowerCamelCase           =					os.path.join(
               A_					,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

           with open(A_					,					'''w'''					,					encoding='''utf-8'''    ) as f:
                 f.write(json.dumps(self.encoder					,					indent=2					,					sort_keys=A_					,					ensure_ascii=A_    ) + '''\n'''    )

           _lowerCamelCase           =					0
           with open(A_					,					'''w'''					,					encoding='''utf-8'''    ) as writer:
                 writer.write('''#version: 0.2\n'''    )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,					key=lambda A_    : kv[1]    ):
                       if index != token_index:
                             logger.warning(
                                 F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                 ''' Please check that the tokenizer is not corrupted!'''    )
                             _lowerCamelCase           =					token_index
                       writer.write(''' '''.join(A_    ) + '''\n'''    )
                       index += 1

           return vocab_file, merge_file



     def 		UpperCamelCase_    (       self					,					A_					,					A_ = None    )			->					List[int]:






           """simple docstring"""
           if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           _lowerCamelCase           =					[self.cls_token_id]
           _lowerCamelCase           =					[self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep



     def 		UpperCamelCase_    (       self					,					A_					,					A_ = None					,					A_ = False    )			->					List[int]:






           """simple docstring"""
           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=A_					,					token_ids_a=A_					,					already_has_special_tokens=A_    )

           if token_ids_a is None:
                 return [1] + ([0] * len(A_    )) + [1]
           return [1] + ([0] * len(A_    )) + [1, 1] + ([0] * len(A_    )) + [1]



     def 		UpperCamelCase_    (       self					,					A_					,					A_ = None    )			->					List[int]:






           """simple docstring"""
           _lowerCamelCase           =					[self.sep_token_id]
           _lowerCamelCase           =					[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep    ) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]





     def 		UpperCamelCase_    (       self					,					A_					,					A_=False					,					**A_    )			->					Dict:






           """simple docstring"""
           _lowerCamelCase           =					kwargs.pop('''add_prefix_space'''					,					self.add_prefix_space    )
           if (is_split_into_words or add_prefix_space) and (len(A_    ) > 0 and not text[0].isspace()):
                 _lowerCamelCase           =					''' ''' + text
           return (text, kwargs)