


'''simple docstring'''




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
				from tokenizers import Tokenizer
				from tokenizers.models import WordPiece



class 			lowerCAmelCase					(					unittest.TestCase				):



	def 					lowercase  (       self			,       snake_case__     ):
		lowerCAmelCase       :     Optional[Any]   				=			list(tokenizer_class.max_model_input_sizes.keys()     )
		for model_name in sa_models[:1]:
			lowerCAmelCase       :     int   				=			tokenizer_class.from_pretrained(snake_case__     )
			self.assertIsNotNone(snake_case__     )
			self.assertIsInstance(snake_case__			,       snake_case__     )
			self.assertIsInstance(snake_case__			,       snake_case__     )

			for special_tok in tokenizer.all_special_tokens:
				self.assertIsInstance(snake_case__			,       snake_case__     )
				lowerCAmelCase       :     List[Any]   				=			tokenizer.convert_tokens_to_ids(snake_case__     )
				self.assertIsInstance(snake_case__			,       snake_case__     )



	def 					lowercase  (       self			,       snake_case__			,       snake_case__ = None     ):
		lowerCAmelCase       :     Optional[int]   				=			pickle.dumps(snake_case__     )
		self.assertIsNotNone(snake_case__     )

		lowerCAmelCase       :     Optional[Any]   				=			pickle.loads(snake_case__     )

		# Ensure is_fast is correctly restored
		self.assertEqual(be_restored.is_fast			,       be_original.is_fast     )

		# Ensure encodings are potentially correctly restored
		if be_original.is_fast:
			self.assertIsNotNone(be_restored.encodings     )
		else:
			self.assertIsNone(be_restored.encodings     )

		# Ensure the keys are the same
		for original_v, restored_v in zip(be_original.values()			,       be_restored.values()     ):
			if equal_op:
				self.assertTrue(equal_op(snake_case__			,       snake_case__     )     )
			else:
				self.assertEqual(snake_case__			,       snake_case__     )



	@slow
	def 					lowercase  (       self     ):
		self.check_tokenizer_from_pretrained(snake_case__     )



	def 					lowercase  (       self     ):
		self.assertEqual(TensorType('tf'     )			,       TensorType.TENSORFLOW     )
		self.assertEqual(TensorType('pt'     )			,       TensorType.PYTORCH     )
		self.assertEqual(TensorType('np'     )			,       TensorType.NUMPY     )



	@require_tokenizers
	def 					lowercase  (       self     ):
		import numpy as np

		lowerCAmelCase       :     int   				=			BertTokenizer.from_pretrained('bert-base-cased'     )
		lowerCAmelCase       :     Optional[Any]   				=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

		# Python no tensor
		with self.subTest('BatchEncoding (Python, return_tensors=None)'     ):
			self.assert_dump_and_restore(tokenizer_p('Small example to encode'     )     )

		with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'     ):
			self.assert_dump_and_restore(
			    tokenizer_p('Small example to encode'			,       return_tensors=TensorType.NUMPY     )			,       np.array_equal     )

		with self.subTest('BatchEncoding (Rust, return_tensors=None)'     ):
			self.assert_dump_and_restore(tokenizer_r('Small example to encode'     )     )

		with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'     ):
			self.assert_dump_and_restore(
			    tokenizer_r('Small example to encode'			,       return_tensors=TensorType.NUMPY     )			,       np.array_equal     )



	@require_tf
	@require_tokenizers
	def 					lowercase  (       self     ):
		import tensorflow as tf

		def tf_array_equals(snake_case__			,       snake_case__     ):
			return tf.reduce_all(tf.equal(snake_case__			,       snake_case__     )     )

		lowerCAmelCase       :     Optional[int]   				=			BertTokenizer.from_pretrained('bert-base-cased'     )
		lowerCAmelCase       :     List[str]   				=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

		with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'     ):
			self.assert_dump_and_restore(
			    tokenizer_p('Small example to encode'			,       return_tensors=TensorType.TENSORFLOW     )			,       snake_case__     )

		with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'     ):
			self.assert_dump_and_restore(
			    tokenizer_r('Small example to encode'			,       return_tensors=TensorType.TENSORFLOW     )			,       snake_case__     )



	@require_torch
	@require_tokenizers
	def 					lowercase  (       self     ):
		import torch

		lowerCAmelCase       :     Tuple   				=			BertTokenizer.from_pretrained('bert-base-cased'     )
		lowerCAmelCase       :     List[Any]   				=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

		with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'     ):
			self.assert_dump_and_restore(
			    tokenizer_p('Small example to encode'			,       return_tensors=TensorType.PYTORCH     )			,       torch.equal     )

		with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'     ):
			self.assert_dump_and_restore(
			    tokenizer_r('Small example to encode'			,       return_tensors=TensorType.PYTORCH     )			,       torch.equal     )



	@require_tokenizers
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     str   				=			BertTokenizer.from_pretrained('bert-base-cased'     )
		lowerCAmelCase       :     List[Any]   				=			BertTokenizerFast.from_pretrained('bert-base-cased'     )

		with self.subTest('Python Tokenizer'     ):
			self.assertFalse(tokenizer_p('Small example to_encode'     ).is_fast     )

		with self.subTest('Rust Tokenizer'     ):
			self.assertTrue(tokenizer_r('Small example to_encode'     ).is_fast     )



	@require_tokenizers
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     str   				=			BertTokenizerFast.from_pretrained('bert-base-cased'     )
		lowerCAmelCase       :     Tuple   				=			tokenizer_r(['Test', '\xad', 'test']			,       is_split_into_words=snake_case__     )

		self.assertEqual(encoded.word_to_tokens(0     )			,       TokenSpan(start=1			,       end=2     )     )
		self.assertEqual(encoded.word_to_tokens(1     )			,       snake_case__     )
		self.assertEqual(encoded.word_to_tokens(2     )			,       TokenSpan(start=2			,       end=3     )     )



	def 					lowercase  (       self     ):
		lowerCAmelCase       :     List[Any]   				=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
		lowerCAmelCase       :     List[Any]   				=			batch.convert_to_tensors(tensor_type='np'     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (2, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (2,)     )
		# test converting the converted
		with CaptureStderr() as cs:
			lowerCAmelCase       :     Union[str, Any]   				=			batch.convert_to_tensors(tensor_type='np'     )
		self.assertFalse(len(cs.err     )			,       msg=f"should have no warning, but got {cs.err}"     )

		lowerCAmelCase       :     List[Any]   				=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
		lowerCAmelCase       :     List[str]   				=			batch.convert_to_tensors(tensor_type='np'			,       prepend_batch_axis=snake_case__     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (1, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (1,)     )



	@require_torch
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     List[str]   				=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
		lowerCAmelCase       :     int   				=			batch.convert_to_tensors(tensor_type='pt'     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (2, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (2,)     )
		# test converting the converted
		with CaptureStderr() as cs:
			lowerCAmelCase       :     int   				=			batch.convert_to_tensors(tensor_type='pt'     )
		self.assertFalse(len(cs.err     )			,       msg=f"should have no warning, but got {cs.err}"     )

		lowerCAmelCase       :     int   				=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
		lowerCAmelCase       :     Union[str, Any]   				=			batch.convert_to_tensors(tensor_type='pt'			,       prepend_batch_axis=snake_case__     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (1, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (1,)     )



	@require_tf
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     List[str]   				=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
		lowerCAmelCase       :     str   				=			batch.convert_to_tensors(tensor_type='tf'     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (2, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (2,)     )
		# test converting the converted
		with CaptureStderr() as cs:
			lowerCAmelCase       :     int   				=			batch.convert_to_tensors(tensor_type='tf'     )
		self.assertFalse(len(cs.err     )			,       msg=f"should have no warning, but got {cs.err}"     )

		lowerCAmelCase       :     Union[str, Any]   				=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
		lowerCAmelCase       :     Dict   				=			batch.convert_to_tensors(tensor_type='tf'			,       prepend_batch_axis=snake_case__     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (1, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (1,)     )



	@require_flax
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     Optional[int]   				=			BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}     )
		lowerCAmelCase       :     Optional[Any]   				=			batch.convert_to_tensors(tensor_type='jax'     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (2, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (2,)     )
		# test converting the converted
		with CaptureStderr() as cs:
			lowerCAmelCase       :     Tuple   				=			batch.convert_to_tensors(tensor_type='jax'     )
		self.assertFalse(len(cs.err     )			,       msg=f"should have no warning, but got {cs.err}"     )

		lowerCAmelCase       :     Dict   				=			BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}     )
		lowerCAmelCase       :     List[Any]   				=			batch.convert_to_tensors(tensor_type='jax'			,       prepend_batch_axis=snake_case__     )
		self.assertEqual(tensor_batch['inputs'].shape			,       (1, 3)     )
		self.assertEqual(tensor_batch['labels'].shape			,       (1,)     )



	def 					lowercase  (       self     ):
		lowerCAmelCase       :     str   				=			[{'input_ids': np.array([0, 1, 2]     )}, {'input_ids': np.array([0, 1, 2, 3]     )}]
		lowerCAmelCase       :     Optional[int]   				=			BertTokenizer.from_pretrained('bert-base-cased'     )

		lowerCAmelCase       :     Any   				=			tokenizer.pad(snake_case__			,       padding=snake_case__     )
		self.assertTrue(isinstance(batch['input_ids']			,       np.ndarray     )     )
		self.assertEqual(batch['input_ids'].tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
		lowerCAmelCase       :     Union[str, Any]   				=			tokenizer.pad(snake_case__			,       padding=snake_case__			,       return_tensors='np'     )
		self.assertTrue(isinstance(batch['input_ids']			,       np.ndarray     )     )
		self.assertEqual(batch['input_ids'].tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )



	@require_torch
	def 					lowercase  (       self     ):
		import torch

		lowerCAmelCase       :     Optional[int]   				=			[{'input_ids': torch.tensor([0, 1, 2]     )}, {'input_ids': torch.tensor([0, 1, 2, 3]     )}]
		lowerCAmelCase       :     List[str]   				=			BertTokenizer.from_pretrained('bert-base-cased'     )

		lowerCAmelCase       :     int   				=			tokenizer.pad(snake_case__			,       padding=snake_case__     )
		self.assertTrue(isinstance(batch['input_ids']			,       torch.Tensor     )     )
		self.assertEqual(batch['input_ids'].tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
		lowerCAmelCase       :     Any   				=			tokenizer.pad(snake_case__			,       padding=snake_case__			,       return_tensors='pt'     )
		self.assertTrue(isinstance(batch['input_ids']			,       torch.Tensor     )     )
		self.assertEqual(batch['input_ids'].tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )



	@require_tf
	def 					lowercase  (       self     ):
		import tensorflow as tf

		lowerCAmelCase       :     str   				=			[{'input_ids': tf.constant([0, 1, 2]     )}, {'input_ids': tf.constant([0, 1, 2, 3]     )}]
		lowerCAmelCase       :     int   				=			BertTokenizer.from_pretrained('bert-base-cased'     )

		lowerCAmelCase       :     Dict   				=			tokenizer.pad(snake_case__			,       padding=snake_case__     )
		self.assertTrue(isinstance(batch['input_ids']			,       tf.Tensor     )     )
		self.assertEqual(batch['input_ids'].numpy().tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
		lowerCAmelCase       :     int   				=			tokenizer.pad(snake_case__			,       padding=snake_case__			,       return_tensors='tf'     )
		self.assertTrue(isinstance(batch['input_ids']			,       tf.Tensor     )     )
		self.assertEqual(batch['input_ids'].numpy().tolist()			,       [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )



	@require_tokenizers
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     Optional[int]   				=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
		PreTrainedTokenizerFast(tokenizer_object=snake_case__     )



	@require_tokenizers
	def 					lowercase  (       self     ):
		lowerCAmelCase       :     int   				=			Tokenizer(WordPiece(unk_token='[UNK]'     )     )
		with tempfile.TemporaryDirectory() as tmpdirname:
			bert_tokenizer.save(os.path.join(snake_case__			,       'tokenizer.json'     )     )
			PreTrainedTokenizerFast(tokenizer_file=os.path.join(snake_case__			,       'tokenizer.json'     )     )




'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase	:		Tuple		  =						logging.get_logger(__name__)

_lowerCAmelCase	:		Dict		  =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCAmelCase	:		Optional[int]		  =						{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

_lowerCAmelCase	:		Tuple		  =						{
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}

@lru_cache()
def       __UpperCamelCase     (     )     ->   Any:




	"""simple docstring"""





	lowerCAmelCase       :     Any   				=			(
	    list(range(ord('!'					)       , ord('~'					) + 1					)					) + list(range(ord('¡'					)       , ord('¬'					) + 1					)					) + list(range(ord('®'					)       , ord('ÿ'					) + 1					)					)
	)
	lowerCAmelCase       :     Optional[int]   				=			bs[:]
	lowerCAmelCase       :     Optional[int]   				=			0
	for b in range(2**8					):
		if b not in bs:
			bs.append(_A					)
			cs.append(2**8 + n					)
			n += 1
	lowerCAmelCase       :     int   				=			[chr(_A					) for n in cs]
	return dict(zip(_A       , _A					)					)

def       __UpperCamelCase     (     _A     :					Union[str, Any]					)     ->   Any:




	"""simple docstring"""





	lowerCAmelCase       :     Tuple   				=			set()
	lowerCAmelCase       :     List[Any]   				=			word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)					)
		lowerCAmelCase       :     Dict   				=			char
	return pairs






class 			lowerCAmelCase					(					a				):
	_lowerCamelCase :       int         =			VOCAB_FILES_NAMES
	_lowerCamelCase :       Any         =			PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase :       int         =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase :       Optional[Any]         =			["""input_ids""", """attention_mask"""]



	def __init__(       self			,       snake_case__			,       snake_case__			,       snake_case__="replace"			,       snake_case__="<s>"			,       snake_case__="</s>"			,       snake_case__="</s>"			,       snake_case__="<s>"			,       snake_case__="<unk>"			,       snake_case__="<pad>"			,       snake_case__="<mask>"			,       snake_case__=False			,       **snake_case__			,       ):
		lowerCAmelCase       :     Optional[Any]   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else bos_token
		lowerCAmelCase       :     List[Any]   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else eos_token
		lowerCAmelCase       :     Any   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else sep_token
		lowerCAmelCase       :     str   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else cls_token
		lowerCAmelCase       :     Tuple   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else unk_token
		lowerCAmelCase       :     List[Any]   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else pad_token

		# Mask token behave like a normal word, i.e. include the space before it
		lowerCAmelCase       :     List[str]   				=			AddedToken(snake_case__			,       lstrip=snake_case__			,       rstrip=snake_case__     ) if isinstance(snake_case__			,       snake_case__     ) else mask_token

		super().__init__(
		    errors=snake_case__			,       bos_token=snake_case__			,       eos_token=snake_case__			,       unk_token=snake_case__			,       sep_token=snake_case__			,       cls_token=snake_case__			,       pad_token=snake_case__			,       mask_token=snake_case__			,       add_prefix_space=snake_case__			,       **snake_case__			,       )

		with open(snake_case__			,       encoding='utf-8'     ) as vocab_handle:
			lowerCAmelCase       :     Tuple   				=			json.load(snake_case__     )
		lowerCAmelCase       :     Union[str, Any]   				=			{v: k for k, v in self.encoder.items()}
		lowerCAmelCase       :     List[str]   				=			errors  # how to handle errors in decoding
		lowerCAmelCase       :     Tuple   				=			bytes_to_unicode()
		lowerCAmelCase       :     Dict   				=			{v: k for k, v in self.byte_encoder.items()}
		with open(snake_case__			,       encoding='utf-8'     ) as merges_handle:
			lowerCAmelCase       :     List[str]   				=			merges_handle.read().split('\n'     )[1:-1]
		lowerCAmelCase       :     Tuple   				=			[tuple(merge.split()     ) for merge in bpe_merges]
		lowerCAmelCase       :     Optional[Any]   				=			dict(zip(snake_case__			,       range(len(snake_case__     )     )     )     )
		lowerCAmelCase       :     List[str]   				=			{}
		lowerCAmelCase       :     Dict   				=			add_prefix_space

		# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
		lowerCAmelCase       :     Dict   				=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'     )



	@property
	def 					lowercase  (       self     ):
		return len(self.encoder     )



	def 					lowercase  (       self     ):
		return dict(self.encoder			,       **self.added_tokens_encoder     )



	def 					lowercase  (       self			,       snake_case__     ):
		if token in self.cache:
			return self.cache[token]
		lowerCAmelCase       :     List[str]   				=			tuple(snake_case__     )
		lowerCAmelCase       :     Union[str, Any]   				=			get_pairs(snake_case__     )

		if not pairs:
			return token

		while True:
			lowerCAmelCase       :     str   				=			min(snake_case__			,       key=lambda snake_case__     : self.bpe_ranks.get(snake_case__			,       float('inf'     )     )     )
			if bigram not in self.bpe_ranks:
				break
			lowerCAmelCase				, lowerCAmelCase       :     List[Any]   				=			bigram
			lowerCAmelCase       :     Optional[int]   				=			[]
			lowerCAmelCase       :     Union[str, Any]   				=			0
			while i < len(snake_case__     ):
				try:
					lowerCAmelCase       :     Union[str, Any]   				=			word.index(snake_case__			,       snake_case__     )
				except ValueError:
					new_word.extend(word[i:]     )
					break
				else:
					new_word.extend(word[i:j]     )
					lowerCAmelCase       :     List[str]   				=			j

				if word[i] == first and i < len(snake_case__     ) - 1 and word[i + 1] == second:
					new_word.append(first + second     )
					i += 2
				else:
					new_word.append(word[i]     )
					i += 1
			lowerCAmelCase       :     int   				=			tuple(snake_case__     )
			lowerCAmelCase       :     List[Any]   				=			new_word
			if len(snake_case__     ) == 1:
				break
			else:
				lowerCAmelCase       :     Union[str, Any]   				=			get_pairs(snake_case__     )
		lowerCAmelCase       :     Optional[Any]   				=			' '.join(snake_case__     )
		lowerCAmelCase       :     Any   				=			word
		return word



	def 					lowercase  (       self			,       snake_case__     ):
		lowerCAmelCase       :     List[Any]   				=			[]
		for token in re.findall(self.pat			,       snake_case__     ):
			lowerCAmelCase       :     Dict   				=			''.join(
			    self.byte_encoder[b] for b in token.encode('utf-8'     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__     ).split(' '     )     )
		return bpe_tokens



	def 					lowercase  (       self			,       snake_case__     ):
		return self.encoder.get(snake_case__			,       self.encoder.get(self.unk_token     )     )



	def 					lowercase  (       self			,       snake_case__     ):
		return self.decoder.get(snake_case__     )



	def 					lowercase  (       self			,       snake_case__     ):
		lowerCAmelCase       :     Union[str, Any]   				=			''.join(snake_case__     )
		lowerCAmelCase       :     Any   				=			bytearray([self.byte_decoder[c] for c in text]     ).decode('utf-8'			,       errors=self.errors     )
		return text



	def 					lowercase  (       self			,       snake_case__			,       snake_case__ = None     ):
		if not os.path.isdir(snake_case__     ):
			logger.error(f"Vocabulary path ({save_directory}) should be a directory"     )
			return
		lowerCAmelCase       :     Union[str, Any]   				=			os.path.join(
		    snake_case__			,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )
		lowerCAmelCase       :     int   				=			os.path.join(
		    snake_case__			,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']     )

		with open(snake_case__			,       'w'			,       encoding='utf-8'     ) as f:
			f.write(json.dumps(self.encoder			,       indent=2			,       sort_keys=snake_case__			,       ensure_ascii=snake_case__     ) + '\n'     )

		lowerCAmelCase       :     List[Any]   				=			0
		with open(snake_case__			,       'w'			,       encoding='utf-8'     ) as writer:
			writer.write('#version: 0.2\n'     )
			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,       key=lambda snake_case__     : kv[1]     ):
				if index != token_index:
					logger.warning(
					    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
					    ' Please check that the tokenizer is not corrupted!'     )
					lowerCAmelCase       :     int   				=			token_index
				writer.write(' '.join(snake_case__     ) + '\n'     )
				index += 1

		return vocab_file, merge_file



	def 					lowercase  (       self			,       snake_case__			,       snake_case__ = None     ):
		if token_ids_a is None:
			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
		lowerCAmelCase       :     Optional[Any]   				=			[self.cls_token_id]
		lowerCAmelCase       :     List[str]   				=			[self.sep_token_id]
		return cls + token_ids_a + sep + sep + token_ids_a + sep



	def 					lowercase  (       self			,       snake_case__			,       snake_case__ = None			,       snake_case__ = False     ):
		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=snake_case__			,       token_ids_a=snake_case__			,       already_has_special_tokens=snake_case__     )

		if token_ids_a is None:
			return [1] + ([0] * len(snake_case__     )) + [1]
		return [1] + ([0] * len(snake_case__     )) + [1, 1] + ([0] * len(snake_case__     )) + [1]



	def 					lowercase  (       self			,       snake_case__			,       snake_case__ = None     ):
		lowerCAmelCase       :     Union[str, Any]   				=			[self.sep_token_id]
		lowerCAmelCase       :     Tuple   				=			[self.cls_token_id]

		if token_ids_a is None:
			return len(cls + token_ids_a + sep     ) * [0]
		return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]



	def 					lowercase  (       self			,       snake_case__			,       snake_case__=False			,       **snake_case__     ):
		lowerCAmelCase       :     List[str]   				=			kwargs.pop('add_prefix_space'			,       self.add_prefix_space     )
		if (is_split_into_words or add_prefix_space) and (len(snake_case__     ) > 0 and not text[0].isspace()):
			lowerCAmelCase       :     Optional[Any]   				=			' ' + text
		return (text, kwargs)
