




'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_lowerCamelCase				:		Union[str, Any]				      =    logging.get_logger(__name__)

_lowerCamelCase				:		Tuple				      =    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase				:		Union[str, Any]				      =    {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_lowerCamelCase				:		Union[str, Any]				      =    {
    'Salesforce/codegen-350M-mono': 2_048,
}






@lru_cache()
def 						_lowercase						(      ):



		'''simple docstring'''







		_snake_case      			=			(
		    list(range(ord("!"    )		,				ord("~"    ) + 1    )    ) + list(range(ord("¡"    )		,				ord("¬"    ) + 1    )    ) + list(range(ord("®"    )		,				ord("ÿ"    ) + 1    )    )
		)
		_snake_case      			=			bs[:]
		_snake_case      			=			0
		for b in range(2**8    ):
				if b not in bs:
						bs.append(lowercase_    )
						cs.append(2**8 + n    )
						n += 1
		_snake_case      			=			[chr(lowercase_    ) for n in cs]
		return dict(zip(lowercase_		,				lowercase_    )    )






def 						_lowercase						(      lowercase_    ):



		'''simple docstring'''







		_snake_case      			=			set()
		_snake_case      			=			word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				_snake_case      			=			char
		return pairs





class 	lowerCamelCase_				(			UpperCAmelCase_				):




					'''simple docstring'''






					lowercase__       =					VOCAB_FILES_NAMES
					lowercase__       =					PRETRAINED_VOCAB_FILES_MAP
					lowercase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__       =					["""input_ids""", """attention_mask"""]






					def __init__(       self,    __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase="replace",    __UpperCamelCase="<|endoftext|>",    __UpperCamelCase="<|endoftext|>",    __UpperCamelCase="<|endoftext|>",    __UpperCamelCase=None,    __UpperCamelCase=False,    __UpperCamelCase=False,    **__UpperCamelCase,    ):
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else bos_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else eos_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else unk_token
							_snake_case      			=			AddedToken(__UpperCamelCase,    lstrip=__UpperCamelCase,    rstrip=__UpperCamelCase					) if isinstance(__UpperCamelCase,    __UpperCamelCase					) else pad_token
							super().__init__(
							    errors=__UpperCamelCase,    unk_token=__UpperCamelCase,    bos_token=__UpperCamelCase,    eos_token=__UpperCamelCase,    pad_token=__UpperCamelCase,    add_prefix_space=__UpperCamelCase,    add_bos_token=__UpperCamelCase,    **__UpperCamelCase,    )
							_snake_case      			=			add_bos_token

							with open(__UpperCamelCase,    encoding="utf-8"					) as vocab_handle:
									_snake_case      			=			json.load(__UpperCamelCase					)
							_snake_case      			=			{v: k for k, v in self.encoder.items()}
							_snake_case      			=			errors  # how to handle errors in decoding
							_snake_case      			=			bytes_to_unicode()
							_snake_case      			=			{v: k for k, v in self.byte_encoder.items()}
							with open(__UpperCamelCase,    encoding="utf-8"					) as merges_handle:
									_snake_case      			=			merges_handle.read().split("\n"					)[1:-1]
							_snake_case      			=			[tuple(merge.split()					) for merge in bpe_merges]
							_snake_case      			=			dict(zip(__UpperCamelCase,    range(len(__UpperCamelCase					)					)					)					)
							_snake_case      			=			{}
							_snake_case      			=			add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							_snake_case      			=			re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"					)






					@property
					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return len(self.encoder					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							return dict(self.encoder,    **self.added_tokens_encoder					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							if token in self.cache:
									return self.cache[token]
							_snake_case      			=			tuple(__UpperCamelCase					)
							_snake_case      			=			get_pairs(__UpperCamelCase					)

							if not pairs:
									return token

							while True:
									_snake_case      			=			min(__UpperCamelCase,    key=lambda __UpperCamelCase					: self.bpe_ranks.get(__UpperCamelCase,    float("inf"					)					)					)
									if bigram not in self.bpe_ranks:
											break
									_snake_case			,       _snake_case      			=			bigram
									_snake_case      			=			[]
									_snake_case      			=			0
									while i < len(__UpperCamelCase					):
											try:
													_snake_case      			=			word.index(__UpperCamelCase,    __UpperCamelCase					)
											except ValueError:
													new_word.extend(word[i:]					)
													break
											else:
													new_word.extend(word[i:j]					)
													_snake_case      			=			j

											if word[i] == first and i < len(__UpperCamelCase					) - 1 and word[i + 1] == second:
													new_word.append(first + second					)
													i += 2
											else:
													new_word.append(word[i]					)
													i += 1
									_snake_case      			=			tuple(__UpperCamelCase					)
									_snake_case      			=			new_word
									if len(__UpperCamelCase					) == 1:
											break
									else:
											_snake_case      			=			get_pairs(__UpperCamelCase					)
							_snake_case      			=			" ".join(__UpperCamelCase					)
							_snake_case      			=			word
							return word






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase=None					):
							if self.add_bos_token:
									_snake_case      			=			[self.bos_token_id]
							else:
									_snake_case      			=			[]

							_snake_case      			=			bos_token_ids + token_ids_a

							if token_ids_a is None:
									return output

							return output + bos_token_ids + token_ids_a






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			[]
							for token in re.findall(self.pat,    __UpperCamelCase					):
									_snake_case      			=			"".join(
									    self.byte_encoder[b] for b in token.encode("utf-8"					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCamelCase					).split(" "					)					)
							return bpe_tokens






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							return self.encoder.get(__UpperCamelCase,    self.encoder.get(self.unk_token					)					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							return self.decoder.get(__UpperCamelCase					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase					):
							_snake_case      			=			"".join(__UpperCamelCase					)
							_snake_case      			=			bytearray([self.byte_decoder[c] for c in text]					).decode("utf-8",    errors=self.errors					)
							return text






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = None					):
							if not os.path.isdir(__UpperCamelCase					):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
									return
							_snake_case      			=			os.path.join(
							    __UpperCamelCase,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
							_snake_case      			=			os.path.join(
							    __UpperCamelCase,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

							with open(__UpperCamelCase,    "w",    encoding="utf-8"					) as f:
									f.write(json.dumps(self.encoder,    indent=2,    sort_keys=__UpperCamelCase,    ensure_ascii=__UpperCamelCase					) + "\n"					)

							_snake_case      			=			0
							with open(__UpperCamelCase,    "w",    encoding="utf-8"					) as writer:
									writer.write("#version: 0.2\n"					)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),    key=lambda __UpperCamelCase					: kv[1]					):
											if index != token_index:
													logger.warning(
													    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
													    " Please check that the tokenizer is not corrupted!"					)
													_snake_case      			=			token_index
											writer.write(" ".join(__UpperCamelCase					) + "\n"					)
											index += 1

							return vocab_file, merge_file






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase=False,    **__UpperCamelCase					):
							_snake_case      			=			kwargs.pop("add_prefix_space",    self.add_prefix_space					)
							if is_split_into_words or add_prefix_space:
									_snake_case      			=			" " + text
							return (text, kwargs)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase = False,    __UpperCamelCase = None,    __UpperCamelCase = None,    **__UpperCamelCase,    ):
							_snake_case      			=			super()._decode(
							    token_ids=__UpperCamelCase,    skip_special_tokens=__UpperCamelCase,    clean_up_tokenization_spaces=__UpperCamelCase,    **__UpperCamelCase,    )

							if truncate_before_pattern is not None and len(__UpperCamelCase					) > 0:
									_snake_case      			=			self.truncate(__UpperCamelCase,    __UpperCamelCase					)

							return decoded_text






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase					):
							def find_re(__UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase					):
									_snake_case      			=			pattern.search(__UpperCamelCase,    __UpperCamelCase					)
									return m.start() if m else -1

							_snake_case      			=			[re.compile(__UpperCamelCase,    re.MULTILINE					) for pattern in truncate_before_pattern]

							_snake_case      			=			list(re.finditer("^print",    __UpperCamelCase,    re.MULTILINE					)					)

							if len(__UpperCamelCase					) > 1:
									_snake_case      			=			completion[: prints[1].start()]

							_snake_case      			=			list(re.finditer("^def",    __UpperCamelCase,    re.MULTILINE					)					)

							if len(__UpperCamelCase					) > 1:
									_snake_case      			=			completion[: defs[1].start()]

							_snake_case      			=			0

							_snake_case      			=			[
							    pos for pos in [find_re(__UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase					) for terminal in terminals] if pos != -1
							]

							if len(__UpperCamelCase					) > 0:
									return completion[: min(__UpperCamelCase					)]
							else:
									return completion











'''simple docstring'''







import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	lowerCamelCase_				(			UpperCAmelCase_,			unittest.TestCase				):




					'''simple docstring'''






					lowercase__       =					PreTrainedTokenizerFast
					lowercase__       =					False
					lowercase__       =					True
					lowercase__       =					"""tokenizer_file"""






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			False  # because we don't have pretrained_vocab_files_map
							super().setUp()
							_snake_case      			=			True

							_snake_case      			=			["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
							_snake_case      			=			"SaulLu/dummy-tokenizer-bytelevel-bpe"

							# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
							_snake_case      			=			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

							_snake_case      			=			PreTrainedTokenizerFast.from_pretrained(model_paths[0]					)
							tokenizer.save_pretrained(self.tmpdirname					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
							# model
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
							pass






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
											try:
													_snake_case      			=			tempfile.mkdtemp()
													_snake_case      			=			self.rust_tokenizer_class.from_pretrained(__UpperCamelCase,    **__UpperCamelCase					)

													tokenizer.save_pretrained(self.tmpdirname					)
													super().test_training_new_tokenizer()
											finally:
													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
													# is restored
													shutil.rmtree(self.tmpdirname					)
													_snake_case      			=			tmpdirname_orig






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.tmpdirname
							# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
											try:
													_snake_case      			=			tempfile.mkdtemp()
													_snake_case      			=			self.rust_tokenizer_class.from_pretrained(__UpperCamelCase,    **__UpperCamelCase					)

													tokenizer.save_pretrained(self.tmpdirname					)
													super().test_training_new_tokenizer_with_special_tokens_change()
											finally:
													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
													# is restored
													shutil.rmtree(self.tmpdirname					)
													_snake_case      			=			tmpdirname_orig






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name					)

							_snake_case      			=			("a" for _ in range(1_000					))
							_snake_case      			=			tokenizer.train_new_from_iterator(text_iterator=__UpperCamelCase,    length=1_000,    vocab_size=50					)

							_snake_case      			=			new_tokenizer.encode("a🤗"					)
							self.assertEqual(__UpperCamelCase,    [64, 172, 253, 97, 245]					)





@require_tokenizers
class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							_snake_case      			=			AutoTokenizer.from_pretrained("bert-base-cased"					)
							_snake_case      			=			json.loads(tokenizer._tokenizer.to_str()					)
							_snake_case      			=			len(__UpperCamelCase					)

							with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									_snake_case      			=			["tokenizer.4.0.0.json"]
									tokenizer.save_pretrained(__UpperCamelCase					)
									json.dump(__UpperCamelCase,    open(os.path.join(__UpperCamelCase,    "tokenizer.4.0.0.json"					),    "w"					)					)

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
									self.assertEqual(len(__UpperCamelCase					),    len(__UpperCamelCase					) + 1					)
									_snake_case      			=			json.loads(new_tokenizer._tokenizer.to_str()					)
									self.assertIn("huggingface",    json_tokenizer["model"]["vocab"]					)

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(__UpperCamelCase,    "tokenizer.4.0.0.json"					),    os.path.join(__UpperCamelCase,    "tokenizer.42.0.0.json"					)					)
									_snake_case      			=			["tokenizer.42.0.0.json"]
									tokenizer.save_pretrained(__UpperCamelCase					)
									_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
									self.assertEqual(len(__UpperCamelCase					),    len(__UpperCamelCase					)					)
									_snake_case      			=			json.loads(new_tokenizer._tokenizer.to_str()					)
									self.assertNotIn("huggingface",    json_tokenizer["model"]["vocab"]					)






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
							_snake_case      			=			"hf-internal-testing/test-two-tokenizers"

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							_snake_case      			=			AutoTokenizer.from_pretrained(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    28_997					)
							_snake_case      			=			json.loads(tokenizer._tokenizer.to_str()					)
							self.assertIn("huggingface",    json_tokenizer["model"]["vocab"]					)

							# Testing an older version by monkey-patching the version in the module it's used.
							import transformers as old_transformers

							_snake_case      			=			"3.0.0"
							_snake_case      			=			old_transformers.models.auto.AutoTokenizer.from_pretrained(__UpperCamelCase					)
							self.assertEqual(len(__UpperCamelCase					),    28_996					)
							_snake_case      			=			json.loads(old_tokenizer._tokenizer.to_str()					)
							self.assertNotIn("huggingface",    json_tokenizer["model"]["vocab"]					)





@require_tokenizers
class 	lowerCamelCase_				(			unittest.TestCase				):




					'''simple docstring'''






					def 							__SCREAMING_SNAKE_CASE      (       self					):
							# See https://github.com/huggingface/transformers/pull/12550
							# and https://github.com/huggingface/tokenizers/issues/537
							_snake_case      			=			PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"					)
							_snake_case      			=			"The Matrix is a 1999 science fiction action film."

							with concurrent.futures.ThreadPoolExecutor() as executor:
									_snake_case      			=			[executor.submit(self.fetch,    __UpperCamelCase,    __UpperCamelCase					) for i in range(10					)]
									_snake_case      			=			[future.result() for future in futures]
									self.assertEqual(__UpperCamelCase,    [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10					)]					)






					def 							__SCREAMING_SNAKE_CASE      (       self,    __UpperCamelCase,    __UpperCamelCase					):
							return tokenizer.encode(__UpperCamelCase,    truncation="longest_first",    padding="longest"					)





