





import collections
import inspect
import unittest

from transformers import NatConfig
from transformers.testing_utils import require_natten, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch
				from torch import nn

				from transformers import NatBackbone, NatForImageClassification, NatModel
				from transformers.models.nat.modeling_nat import NAT_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
				from PIL import Image

				from transformers import AutoImageProcessor




class 	_UpperCAmelCase						:




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase=13     ,   __lowerCAmelCase=64     ,   __lowerCAmelCase=4     ,   __lowerCAmelCase=3     ,   __lowerCAmelCase=16     ,   __lowerCAmelCase=[1, 2, 1]     ,   __lowerCAmelCase=[2, 4, 8]     ,   __lowerCAmelCase=3     ,   __lowerCAmelCase=2.0     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=0.0     ,   __lowerCAmelCase=0.0     ,   __lowerCAmelCase=0.1     ,   __lowerCAmelCase="gelu"     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=0.0_2     ,   __lowerCAmelCase=1E-5     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=None     ,   __lowerCAmelCase=True     ,   __lowerCAmelCase=10     ,   __lowerCAmelCase=["stage1", "stage2"]     ,   __lowerCAmelCase=[1, 2]     ,   )    ->	Optional[int]:
							UpperCamelCase__								=						parent
							UpperCamelCase__								=						batch_size
							UpperCamelCase__								=						image_size
							UpperCamelCase__								=						patch_size
							UpperCamelCase__								=						num_channels
							UpperCamelCase__								=						embed_dim
							UpperCamelCase__								=						depths
							UpperCamelCase__								=						num_heads
							UpperCamelCase__								=						kernel_size
							UpperCamelCase__								=						mlp_ratio
							UpperCamelCase__								=						qkv_bias
							UpperCamelCase__								=						hidden_dropout_prob
							UpperCamelCase__								=						attention_probs_dropout_prob
							UpperCamelCase__								=						drop_path_rate
							UpperCamelCase__								=						hidden_act
							UpperCamelCase__								=						patch_norm
							UpperCamelCase__								=						layer_norm_eps
							UpperCamelCase__								=						initializer_range
							UpperCamelCase__								=						is_training
							UpperCamelCase__								=						scope
							UpperCamelCase__								=						use_labels
							UpperCamelCase__								=						num_labels
							UpperCamelCase__								=						out_features
							UpperCamelCase__								=						out_indices




					def 						snake_case_		(    self							)    ->	List[Any]:
							UpperCamelCase__								=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

							UpperCamelCase__								=						None
							if self.use_labels:
									UpperCamelCase__								=						ids_tensor([self.batch_size]     ,   self.num_labels							)

							UpperCamelCase__								=						self.get_config()

							return config, pixel_values, labels




					def 						snake_case_		(    self							)    ->	List[Any]:
							return NatConfig(
							    num_labels=self.num_labels     ,   image_size=self.image_size     ,   patch_size=self.patch_size     ,   num_channels=self.num_channels     ,   embed_dim=self.embed_dim     ,   depths=self.depths     ,   num_heads=self.num_heads     ,   kernel_size=self.kernel_size     ,   mlp_ratio=self.mlp_ratio     ,   qkv_bias=self.qkv_bias     ,   hidden_dropout_prob=self.hidden_dropout_prob     ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,   drop_path_rate=self.drop_path_rate     ,   hidden_act=self.hidden_act     ,   patch_norm=self.patch_norm     ,   layer_norm_eps=self.layer_norm_eps     ,   initializer_range=self.initializer_range     ,   out_features=self.out_features     ,   out_indices=self.out_indices     ,   )




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Tuple:
							UpperCamelCase__								=						NatModel(config=__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()
							UpperCamelCase__								=						model(__lowerCAmelCase							)

							UpperCamelCase__								=						UpperCamelCase__								=						(config.image_size // config.patch_size) // (2 ** (len(config.depths							) - 1))
							UpperCamelCase__								=						int(config.embed_dim * 2 ** (len(config.depths							) - 1)							)

							self.parent.assertEqual(
							    result.last_hidden_state.shape     ,   (self.batch_size, expected_height, expected_width, expected_dim)							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Union[str, Any]:
							UpperCamelCase__								=						NatForImageClassification(__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()
							UpperCamelCase__								=						model(__lowerCAmelCase     ,   labels=__lowerCAmelCase							)
							self.parent.assertEqual(result.logits.shape     ,   (self.batch_size, self.num_labels)							)

							# test greyscale images
							UpperCamelCase__								=						1
							UpperCamelCase__								=						NatForImageClassification(__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()

							UpperCamelCase__								=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
							UpperCamelCase__								=						model(__lowerCAmelCase							)
							self.parent.assertEqual(result.logits.shape     ,   (self.batch_size, self.num_labels)							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Optional[int]:
							UpperCamelCase__								=						NatBackbone(config=__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()
							UpperCamelCase__								=						model(__lowerCAmelCase							)

							# verify hidden states
							self.parent.assertEqual(len(result.feature_maps							)     ,   len(config.out_features							)							)
							self.parent.assertListEqual(list(result.feature_maps[0].shape							)     ,   [self.batch_size, model.channels[0], 16, 16]							)

							# verify channels
							self.parent.assertEqual(len(model.channels							)     ,   len(config.out_features							)							)

							# verify backbone works with out_features=None
							UpperCamelCase__								=						None
							UpperCamelCase__								=						NatBackbone(config=__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()
							UpperCamelCase__								=						model(__lowerCAmelCase							)

							# verify feature maps
							self.parent.assertEqual(len(result.feature_maps							)     ,   1							)
							self.parent.assertListEqual(list(result.feature_maps[0].shape							)     ,   [self.batch_size, model.channels[-1], 4, 4]							)

							# verify channels
							self.parent.assertEqual(len(model.channels							)     ,   1							)




					def 						snake_case_		(    self							)    ->	Any:
							UpperCamelCase__								=						self.prepare_config_and_inputs()
							UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__								=						config_and_inputs
							UpperCamelCase__								=						{"""pixel_values""": pixel_values}
							return config, inputs_dict




@require_natten
@require_torch
class 	_UpperCAmelCase						( _lowercase			,							_lowercase			,							unittest.TestCase       ):
					__A        = (
					    (
					        NatModel,
					        NatForImageClassification,
					        NatBackbone,
					    )
					    if is_torch_available()
					    else ()
					)
					__A        = (
					    {"feature-extraction": NatModel, "image-classification": NatForImageClassification}
					    if is_torch_available()
					    else {}
					)
					__A        = False

					__A        = False
					__A        = False
					__A        = False
					__A        = False




					def 						snake_case_		(    self							)    ->	Optional[int]:
							UpperCamelCase__								=						NatModelTester(self							)
							UpperCamelCase__								=						ConfigTester(self     ,   config_class=__lowerCAmelCase     ,   embed_dim=37							)




					def 						snake_case_		(    self							)    ->	Any:
							self.create_and_test_config_common_properties()
							self.config_tester.create_and_test_config_to_json_string()
							self.config_tester.create_and_test_config_to_json_file()
							self.config_tester.create_and_test_config_from_and_save_pretrained()
							self.config_tester.create_and_test_config_with_num_labels()
							self.config_tester.check_config_can_be_init_without_params()
							self.config_tester.check_config_arguments_init()




					def 						snake_case_		(    self							)    ->	Optional[int]:
							return




					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__lowerCAmelCase							)




					def 						snake_case_		(    self							)    ->	List[Any]:
							UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_image_classification(*__lowerCAmelCase							)




					def 						snake_case_		(    self							)    ->	Dict:
							UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_backbone(*__lowerCAmelCase							)




					@unittest.skip(reason="""Nat does not use inputs_embeds"""							)
					def 						snake_case_		(    self							)    ->	Any:
							pass




					@unittest.skip(reason="""Nat does not use feedforward chunking"""							)
					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							pass




					def 						snake_case_		(    self							)    ->	int:
							UpperCamelCase__	,      UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
									UpperCamelCase__								=						model_class(__lowerCAmelCase							)
									self.assertIsInstance(model.get_input_embeddings()     ,   (nn.Module)							)
									UpperCamelCase__								=						model.get_output_embeddings()
									self.assertTrue(x is None or isinstance(__lowerCAmelCase     ,   nn.Linear							)							)




					def 						snake_case_		(    self							)    ->	Dict:
							UpperCamelCase__	,      UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
									UpperCamelCase__								=						model_class(__lowerCAmelCase							)
									UpperCamelCase__								=						inspect.signature(model.forward							)
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									UpperCamelCase__								=						[*signature.parameters.keys()]

									UpperCamelCase__								=						["""pixel_values"""]
									self.assertListEqual(arg_names[:1]     ,   __lowerCAmelCase							)




					def 						snake_case_		(    self							)    ->	Any:
							self.skipTest("""Nat's attention operation is handled entirely by NATTEN."""							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)    ->	Tuple:
							UpperCamelCase__								=						model_class(__lowerCAmelCase							)
							model.to(__lowerCAmelCase							)
							model.eval()

							with torch.no_grad():
									UpperCamelCase__								=						model(**self._prepare_for_class(__lowerCAmelCase     ,   __lowerCAmelCase							)							)

							UpperCamelCase__								=						outputs.hidden_states

							UpperCamelCase__								=						getattr(
							    self.model_tester     ,   """expected_num_hidden_layers"""     ,   len(self.model_tester.depths							) + 1							)
							self.assertEqual(len(__lowerCAmelCase							)     ,   __lowerCAmelCase							)

							# Nat has a different seq_length
							UpperCamelCase__								=						(
							    config.patch_size
							    if isinstance(config.patch_size     ,   collections.abc.Iterable							)
							    else (config.patch_size, config.patch_size)
							)

							UpperCamelCase__								=						image_size[0] // patch_size[0]
							UpperCamelCase__								=						image_size[1] // patch_size[1]

							self.assertListEqual(
							    list(hidden_states[0].shape[-3:]							)     ,   [height, width, self.model_tester.embed_dim]     ,   )

							if model_class.__name__ != "NatBackbone":
									UpperCamelCase__								=						outputs.reshaped_hidden_states
									self.assertEqual(len(__lowerCAmelCase							)     ,   __lowerCAmelCase							)

									UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__								=						reshaped_hidden_states[0].shape
									UpperCamelCase__								=						(
									    reshaped_hidden_states[0].view(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							).permute(0     ,   2     ,   3     ,   1							)
									)
									self.assertListEqual(
									    list(reshaped_hidden_states.shape[-3:]							)     ,   [height, width, self.model_tester.embed_dim]     ,   )




					def 						snake_case_		(    self							)    ->	Optional[int]:
							UpperCamelCase__	,      UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs_for_common()

							UpperCamelCase__								=						(
							    self.model_tester.image_size
							    if isinstance(self.model_tester.image_size     ,   collections.abc.Iterable							)
							    else (self.model_tester.image_size, self.model_tester.image_size)
							)

							for model_class in self.all_model_classes:
									UpperCamelCase__								=						True
									self.check_hidden_states_output(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)

									# check that output_hidden_states also work using config
									del inputs_dict["output_hidden_states"]
									UpperCamelCase__								=						True

									self.check_hidden_states_output(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase							)




					@slow
					def 						snake_case_		(    self							)    ->	List[str]:
							for model_name in NAT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									UpperCamelCase__								=						NatModel.from_pretrained(__lowerCAmelCase							)
									self.assertIsNotNone(__lowerCAmelCase							)




					def 						snake_case_		(    self							)    ->	Optional[Any]:
							UpperCamelCase__	,      UpperCamelCase__								=						self.model_tester.prepare_config_and_inputs_for_common()

							UpperCamelCase__								=						_config_zero_init(__lowerCAmelCase							)
							for model_class in self.all_model_classes:
									UpperCamelCase__								=						model_class(config=__lowerCAmelCase							)
									for name, param in model.named_parameters():
											if "embeddings" not in name and param.requires_grad:
													self.assertIn(
													    ((param.data.mean() * 1E9).round() / 1E9).item()     ,   [0.0, 1.0]     ,   msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""     ,   )




@require_natten
@require_vision
@require_torch
class 	_UpperCAmelCase						( unittest.TestCase       ):




					@cached_property
					def 						snake_case_		(    self							)    ->	str:
							return AutoImageProcessor.from_pretrained("""shi-labs/nat-mini-in1k-224"""							) if is_vision_available() else None




					@slow
					def 						snake_case_		(    self							)    ->	str:
							UpperCamelCase__								=						NatForImageClassification.from_pretrained("""shi-labs/nat-mini-in1k-224"""							).to(__lowerCAmelCase							)
							UpperCamelCase__								=						self.default_image_processor

							UpperCamelCase__								=						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""							)
							UpperCamelCase__								=						image_processor(images=__lowerCAmelCase     ,   return_tensors="""pt"""							).to(__lowerCAmelCase							)

							# forward pass
							with torch.no_grad():
									UpperCamelCase__								=						model(**__lowerCAmelCase							)

							# verify the logits
							UpperCamelCase__								=						torch.Size((1, 1_000)							)
							self.assertEqual(outputs.logits.shape     ,   __lowerCAmelCase							)
							UpperCamelCase__								=						torch.tensor([0.3_8_0_5, -0.8_6_7_6, -0.3_9_1_2]							).to(__lowerCAmelCase							)
							self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,   __lowerCAmelCase     ,   atol=1E-4							)							)




@require_torch
@require_natten
class 	_UpperCAmelCase						( unittest.TestCase			,							_lowercase       ):
					__A        = (NatBackbone,) if is_torch_available() else ()
					__A        = NatConfig




					def 						snake_case_		(    self							)    ->	Optional[int]:
							UpperCamelCase__								=						NatModelTester(self							)












import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase 	=      logging.get_logger(__name__)


__lowerCAmelCase 	=      {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all MVP models at https://huggingface.co/models?filter=mvp
__lowerCAmelCase 	=      {
    "vocab_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json",
    },
    "added_tokens.json": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json",
    },
    "merges_file": {
        "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt",
    },
}

__lowerCAmelCase 	=      {
    "RUCAIBox/mvp": 1_024,
}

@lru_cache()
def      a    (   )  ->Any:
		UpperCamelCase__								=						(
		    list(range(ord("""!"""							)			,    ord("""~"""							) + 1							)							) + list(range(ord("""¡"""							)			,    ord("""¬"""							) + 1							)							) + list(range(ord("""®"""							)			,    ord("""ÿ"""							) + 1							)							)
		)
		UpperCamelCase__								=						bs[:]
		UpperCamelCase__								=						0
		for b in range(2**8							):
				if b not in bs:
						bs.append(_lowercase							)
						cs.append(2**8 + n							)
						n += 1
		UpperCamelCase__								=						[chr(_lowercase							) for n in cs]
		return dict(zip(_lowercase			,    _lowercase							)							)

def      a    (   _lowercase :  int							)  ->List[Any]:
		UpperCamelCase__								=						set()
		UpperCamelCase__								=						word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)							)
				UpperCamelCase__								=						char
		return pairs




class 	_UpperCAmelCase						( _lowercase       ):
					__A        = VOCAB_FILES_NAMES
					__A        = PRETRAINED_VOCAB_FILES_MAP
					__A        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__A        = ["input_ids", "attention_mask"]




					def __init__(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase="replace"     ,   __lowerCAmelCase="<s>"     ,   __lowerCAmelCase="</s>"     ,   __lowerCAmelCase="</s>"     ,   __lowerCAmelCase="<s>"     ,   __lowerCAmelCase="<unk>"     ,   __lowerCAmelCase="<pad>"     ,   __lowerCAmelCase="<mask>"     ,   __lowerCAmelCase=False     ,   **__lowerCAmelCase     ,   )    ->	str:
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else bos_token
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else eos_token
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else sep_token
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else cls_token
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else unk_token
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCamelCase__								=						AddedToken(__lowerCAmelCase     ,   lstrip=__lowerCAmelCase     ,   rstrip=__lowerCAmelCase							) if isinstance(__lowerCAmelCase     ,   __lowerCAmelCase							) else mask_token

							super().__init__(
							    errors=__lowerCAmelCase     ,   bos_token=__lowerCAmelCase     ,   eos_token=__lowerCAmelCase     ,   unk_token=__lowerCAmelCase     ,   sep_token=__lowerCAmelCase     ,   cls_token=__lowerCAmelCase     ,   pad_token=__lowerCAmelCase     ,   mask_token=__lowerCAmelCase     ,   add_prefix_space=__lowerCAmelCase     ,   **__lowerCAmelCase     ,   )
							with open(__lowerCAmelCase     ,   encoding="""utf-8"""							) as vocab_handle:
									UpperCamelCase__								=						json.load(__lowerCAmelCase							)
							UpperCamelCase__								=						{v: k for k, v in self.encoder.items()}
							UpperCamelCase__								=						errors  # how to handle errors in decoding
							UpperCamelCase__								=						bytes_to_unicode()
							UpperCamelCase__								=						{v: k for k, v in self.byte_encoder.items()}
							with open(__lowerCAmelCase     ,   encoding="""utf-8"""							) as merges_handle:
									UpperCamelCase__								=						merges_handle.read().split("""\n"""							)[1:-1]
							UpperCamelCase__								=						[tuple(merge.split()							) for merge in bpe_merges]
							UpperCamelCase__								=						dict(zip(__lowerCAmelCase     ,   range(len(__lowerCAmelCase							)							)							)							)
							UpperCamelCase__								=						{}
							UpperCamelCase__								=						add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCamelCase__								=						re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)




					@property
					def 						snake_case_		(    self							)    ->	Union[str, Any]:
							return len(self.encoder							)




					def 						snake_case_		(    self							)    ->	Any:
							return dict(self.encoder     ,   **self.added_tokens_encoder							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	List[str]:
							if token in self.cache:
									return self.cache[token]
							UpperCamelCase__								=						tuple(__lowerCAmelCase							)
							UpperCamelCase__								=						get_pairs(__lowerCAmelCase							)

							if not pairs:
									return token

							while True:
									UpperCamelCase__								=						min(__lowerCAmelCase     ,   key=lambda __lowerCAmelCase							: self.bpe_ranks.get(__lowerCAmelCase     ,   float("""inf"""							)							)							)
									if bigram not in self.bpe_ranks:
											break
									UpperCamelCase__	,      UpperCamelCase__								=						bigram
									UpperCamelCase__								=						[]
									UpperCamelCase__								=						0
									while i < len(__lowerCAmelCase							):
											try:
													UpperCamelCase__								=						word.index(__lowerCAmelCase     ,   __lowerCAmelCase							)
											except ValueError:
													new_word.extend(word[i:]							)
													break
											else:
													new_word.extend(word[i:j]							)
													UpperCamelCase__								=						j

											if word[i] == first and i < len(__lowerCAmelCase							) - 1 and word[i + 1] == second:
													new_word.append(first + second							)
													i += 2
											else:
													new_word.append(word[i]							)
													i += 1
									UpperCamelCase__								=						tuple(__lowerCAmelCase							)
									UpperCamelCase__								=						new_word
									if len(__lowerCAmelCase							) == 1:
											break
									else:
											UpperCamelCase__								=						get_pairs(__lowerCAmelCase							)
							UpperCamelCase__								=						""" """.join(__lowerCAmelCase							)
							UpperCamelCase__								=						word
							return word




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	int:
							UpperCamelCase__								=						[]
							for token in re.findall(self.pat     ,   __lowerCAmelCase							):
									UpperCamelCase__								=						"""""".join(
									    self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase							).split(""" """							)							)
							return bpe_tokens




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Dict:
							return self.encoder.get(__lowerCAmelCase     ,   self.encoder.get(self.unk_token							)							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Optional[Any]:
							return self.decoder.get(__lowerCAmelCase							)




					def 						snake_case_		(    self     ,   __lowerCAmelCase							)    ->	Any:
							UpperCamelCase__								=						"""""".join(__lowerCAmelCase							)
							UpperCamelCase__								=						bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""     ,   errors=self.errors							)
							return text




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase = None							)    ->	Tuple[str]:
							if not os.path.isdir(__lowerCAmelCase							):
									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
									return
							UpperCamelCase__								=						os.path.join(
							    __lowerCAmelCase     ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
							UpperCamelCase__								=						os.path.join(
							    __lowerCAmelCase     ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

							with open(__lowerCAmelCase     ,   """w"""     ,   encoding="""utf-8"""							) as f:
									f.write(json.dumps(self.encoder     ,   indent=2     ,   sort_keys=__lowerCAmelCase     ,   ensure_ascii=__lowerCAmelCase							) + """\n"""							)

							UpperCamelCase__								=						0
							with open(__lowerCAmelCase     ,   """w"""     ,   encoding="""utf-8"""							) as writer:
									writer.write("""#version: 0.2\n"""							)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,   key=lambda __lowerCAmelCase							: kv[1]							):
											if index != token_index:
													logger.warning(
													    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
													    """ Please check that the tokenizer is not corrupted!"""							)
													UpperCamelCase__								=						token_index
											writer.write(""" """.join(__lowerCAmelCase							) + """\n"""							)
											index += 1

							return vocab_file, merge_file




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase = None							)    ->	List[int]:
							if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCamelCase__								=						[self.cls_token_id]
							UpperCamelCase__								=						[self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase = None     ,   __lowerCAmelCase = False							)    ->	List[int]:
							if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__lowerCAmelCase     ,   token_ids_a=__lowerCAmelCase     ,   already_has_special_tokens=__lowerCAmelCase							)

							if token_ids_a is None:
									return [1] + ([0] * len(__lowerCAmelCase							)) + [1]
							return [1] + ([0] * len(__lowerCAmelCase							)) + [1, 1] + ([0] * len(__lowerCAmelCase							)) + [1]




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase = None							)    ->	List[int]:
							UpperCamelCase__								=						[self.sep_token_id]
							UpperCamelCase__								=						[self.cls_token_id]

							if token_ids_a is None:
									return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




					def 						snake_case_		(    self     ,   __lowerCAmelCase     ,   __lowerCAmelCase=False     ,   **__lowerCAmelCase							)    ->	Optional[Any]:
							UpperCamelCase__								=						kwargs.pop("""add_prefix_space"""     ,   self.add_prefix_space							)
							if (is_split_into_words or add_prefix_space) and (len(__lowerCAmelCase							) > 0 and not text[0].isspace()):
									UpperCamelCase__								=						""" """ + text
							return (text, kwargs)





