



"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
				from test_module.custom_tokenization_fast import CustomTokenizerFast







class       A_			(     unittest.TestCase				):



						"""simple docstring"""






						def 	UpperCAmelCase__     (					self :List[str]					)				->			Dict:
												# A mock response for an HTTP head request to emulate server down
												UpperCAmelCase             = mock.Mock()
												UpperCAmelCase             = 5_00
												UpperCAmelCase             = {}
												UpperCAmelCase             = HTTPError
												UpperCAmelCase             = {}

												# Download this model to make sure it's in the cache.
												UpperCAmelCase             = BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'					)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch('requests.Session.request'   ,  return_value=lowercase_					) as mock_head:
																		UpperCAmelCase             = BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'					)
																		# This check we did call the fake head request
																		mock_head.assert_called()






						@require_tokenizers
						def 	UpperCAmelCase__     (					self :List[Any]					)				->			Dict:
												# A mock response for an HTTP head request to emulate server down
												UpperCAmelCase             = mock.Mock()
												UpperCAmelCase             = 5_00
												UpperCAmelCase             = {}
												UpperCAmelCase             = HTTPError
												UpperCAmelCase             = {}

												# Download this model to make sure it's in the cache.
												UpperCAmelCase             = GPTaTokenizerFast.from_pretrained('gpt2'					)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch('requests.Session.request'   ,  return_value=lowercase_					) as mock_head:
																		UpperCAmelCase             = GPTaTokenizerFast.from_pretrained('gpt2'					)
																		# This check we did call the fake head request
																		mock_head.assert_called()






						def 	UpperCAmelCase__     (					self :Any					)				->			Union[str, Any]:
												# This test is for deprecated behavior and can be removed in v5
												try:
																		UpperCAmelCase             = tempfile.mktemp()
																		with open(lowercase_   ,  'wb'					) as f:
																								http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'   ,  lowercase_					)

																		UpperCAmelCase             = AlbertTokenizer.from_pretrained(lowercase_					)
												finally:
																		os.remove(lowercase_					)

												# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
												# the current folder and have the right name.
												if os.path.isfile('tokenizer.json'					):
																		# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																		return
												try:
																		with open('tokenizer.json'   ,  'wb'					) as f:
																								http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'   ,  lowercase_					)
																		UpperCAmelCase             = AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'					)
																		# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																		self.assertEqual(tokenizer.vocab_size   ,  10_00					)
																		# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

												finally:
																		os.remove('tokenizer.json'					)






						def 	UpperCAmelCase__     (					self :Tuple					)				->			Optional[int]:
												# This test is for deprecated behavior and can be removed in v5
												UpperCAmelCase             = AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'					)







@is_staging_test
class       A_			(     unittest.TestCase				):



						"""simple docstring"""
						__UpperCamelCase						=							["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]






						@classmethod
						def 	UpperCAmelCase__     (					cls :Tuple					)				->			List[Any]:
												UpperCAmelCase             = TOKEN
												HfFolder.save_token(lowercase_					)






						@classmethod
						def 	UpperCAmelCase__     (					cls :List[str]					)				->			Optional[int]:
												try:
																		delete_repo(token=cls._token   ,  repo_id='test-tokenizer'					)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,  repo_id='valid_org/test-tokenizer-org'					)
												except HTTPError:
																		pass

												try:
																		delete_repo(token=cls._token   ,  repo_id='test-dynamic-tokenizer'					)
												except HTTPError:
																		pass






						def 	UpperCAmelCase__     (					self :Any					)				->			str:
												with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase             = os.path.join(lowercase_   ,  'vocab.txt'					)
																		with open(lowercase_   ,  'w'   ,  encoding='utf-8'					) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
																		UpperCAmelCase             = BertTokenizer(lowercase_					)

												tokenizer.push_to_hub('test-tokenizer'   ,  use_auth_token=self._token					)
												UpperCAmelCase             = BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""					)
												self.assertDictEqual(new_tokenizer.vocab   ,  tokenizer.vocab					)

												# Reset repo
												delete_repo(token=self._token   ,  repo_id='test-tokenizer'					)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(lowercase_   ,  repo_id='test-tokenizer'   ,  push_to_hub=lowercase_   ,  use_auth_token=self._token					)

												UpperCAmelCase             = BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""					)
												self.assertDictEqual(new_tokenizer.vocab   ,  tokenizer.vocab					)






						def 	UpperCAmelCase__     (					self :int					)				->			Union[str, Any]:
												with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase             = os.path.join(lowercase_   ,  'vocab.txt'					)
																		with open(lowercase_   ,  'w'   ,  encoding='utf-8'					) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
																		UpperCAmelCase             = BertTokenizer(lowercase_					)

												tokenizer.push_to_hub('valid_org/test-tokenizer-org'   ,  use_auth_token=self._token					)
												UpperCAmelCase             = BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'					)
												self.assertDictEqual(new_tokenizer.vocab   ,  tokenizer.vocab					)

												# Reset repo
												delete_repo(token=self._token   ,  repo_id='valid_org/test-tokenizer-org'					)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(
																		    lowercase_   ,  repo_id='valid_org/test-tokenizer-org'   ,  push_to_hub=lowercase_   ,  use_auth_token=self._token					)

												UpperCAmelCase             = BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'					)
												self.assertDictEqual(new_tokenizer.vocab   ,  tokenizer.vocab					)






						@require_tokenizers
						def 	UpperCAmelCase__     (					self :int					)				->			Tuple:
												CustomTokenizer.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase             = os.path.join(lowercase_   ,  'vocab.txt'					)
																		with open(lowercase_   ,  'w'   ,  encoding='utf-8'					) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)
																		UpperCAmelCase             = CustomTokenizer(lowercase_					)

												# No fast custom tokenizer
												tokenizer.push_to_hub('test-dynamic-tokenizer'   ,  use_auth_token=self._token					)

												UpperCAmelCase             = AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""   ,  trust_remote_code=lowercase_					)
												# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,  'CustomTokenizer'					)

												# Fast and slow custom tokenizer
												CustomTokenizerFast.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																		UpperCAmelCase             = os.path.join(lowercase_   ,  'vocab.txt'					)
																		with open(lowercase_   ,  'w'   ,  encoding='utf-8'					) as vocab_writer:
																								vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]					)					)

																		UpperCAmelCase             = BertTokenizerFast.from_pretrained(lowercase_					)
																		bert_tokenizer.save_pretrained(lowercase_					)
																		UpperCAmelCase             = CustomTokenizerFast.from_pretrained(lowercase_					)

												tokenizer.push_to_hub('test-dynamic-tokenizer'   ,  use_auth_token=self._token					)

												UpperCAmelCase             = AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""   ,  trust_remote_code=lowercase_					)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,  'CustomTokenizerFast'					)
												UpperCAmelCase             = AutoTokenizer.from_pretrained(
												    f"""{USER}/test-dynamic-tokenizer"""   ,  use_fast=lowercase_   ,  trust_remote_code=lowercase_					)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__   ,  'CustomTokenizer'					)







class       A_			(     unittest.TestCase				):



						"""simple docstring"""






						def 	UpperCAmelCase__     (					self :List[str]					)				->			Tuple:
												UpperCAmelCase             = Trie()
												trie.add('Hello 友達'					)
												self.assertEqual(trie.data   ,  {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}					)
												trie.add('Hello'					)
												trie.data
												self.assertEqual(trie.data   ,  {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}					)






						def 	UpperCAmelCase__     (					self :Tuple					)				->			str:
												UpperCAmelCase             = Trie()
												self.assertEqual(trie.split('[CLS] This is a extra_id_100'					)   ,  ['[CLS] This is a extra_id_100']					)
												trie.add('[CLS]'					)
												trie.add('extra_id_1'					)
												trie.add('extra_id_100'					)
												self.assertEqual(trie.split('[CLS] This is a extra_id_100'					)   ,  ['[CLS]', ' This is a ', 'extra_id_100']					)






						def 	UpperCAmelCase__     (					self :Union[str, Any]					)				->			Union[str, Any]:
												UpperCAmelCase             = Trie()
												trie.add('A'					)
												self.assertEqual(trie.split('ABC'					)   ,  ['A', 'BC']					)
												self.assertEqual(trie.split('BCA'					)   ,  ['BC', 'A']					)






						def 	UpperCAmelCase__     (					self :Any					)				->			int:
												UpperCAmelCase             = Trie()
												trie.add('TOKEN]'					)
												trie.add('[SPECIAL_TOKEN]'					)
												self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'					)   ,  ['This is something ', '[SPECIAL_TOKEN]']					)






						def 	UpperCAmelCase__     (					self :Any					)				->			Optional[int]:
												UpperCAmelCase             = Trie()
												trie.add('A'					)
												trie.add('P'					)
												trie.add('[SPECIAL_TOKEN]'					)
												self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'					)   ,  ['This is something ', '[SPECIAL_TOKEN]']					)






						def 	UpperCAmelCase__     (					self :int					)				->			Union[str, Any]:
												UpperCAmelCase             = Trie()
												trie.add('AB'					)
												trie.add('B'					)
												trie.add('C'					)
												self.assertEqual(trie.split('ABC'					)   ,  ['AB', 'C']					)






						def 	UpperCAmelCase__     (					self :Optional[int]					)				->			List[Any]:
												UpperCAmelCase             = Trie()
												trie.add('ABC'					)
												trie.add('B'					)
												trie.add('CD'					)
												self.assertEqual(trie.split('ABCD'					)   ,  ['ABC', 'D']					)






						def 	UpperCAmelCase__     (					self :Optional[int]					)				->			Optional[Any]:
												# Even if the offsets are wrong, we necessarily output correct string
												# parts.
												UpperCAmelCase             = Trie()
												UpperCAmelCase             = trie.cut_text('ABC'   ,  [0, 0, 2, 1, 2, 3]					)
												self.assertEqual(lowercase_   ,  ['AB', 'C']					)









import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


__lowerCAmelCase	: Optional[int]  				=logging.get_logger(__name__)

__lowerCAmelCase	: Optional[Any]  				={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	: List[str]  				={
    'tokenizer_file': {
        'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase	: Optional[int]  				={
    'gpt-neox-20b': 2_0_4_8,
}
class        _lowercase      (			A__       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			str						=		VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__				:			Dict						=		PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__				:			Any						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		['''input_ids''', '''attention_mask''']


				def __init__(       self			:int			,			lowerCAmelCase__			:Any=None			,			lowerCAmelCase__			:Optional[Any]=None			,			lowerCAmelCase__			:List[Any]=None			,			lowerCAmelCase__			:str="<|endoftext|>"			,			lowerCAmelCase__			:str="<|endoftext|>"			,			lowerCAmelCase__			:Dict="<|endoftext|>"			,			lowerCAmelCase__			:Union[str, Any]=False			,			**lowerCAmelCase__			:List[str]			,			)		->			Any:
										super().__init__(
										    lowerCAmelCase__			,			lowerCAmelCase__			,			tokenizer_file=lowerCAmelCase__			,			unk_token=lowerCAmelCase__			,			bos_token=lowerCAmelCase__			,			eos_token=lowerCAmelCase__			,			add_prefix_space=lowerCAmelCase__			,			**lowerCAmelCase__			,			)

										__SCREAMING_SNAKE_CASE						:     List[Any]					=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
										if pre_tok_state.get('''add_prefix_space'''			,			lowerCAmelCase__	) != add_prefix_space:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		getattr(lowerCAmelCase__			,			pre_tok_state.pop('''type'''	)	)
																__SCREAMING_SNAKE_CASE						:     str					=		add_prefix_space
																__SCREAMING_SNAKE_CASE						:     Any					=		pre_tok_class(**lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		add_prefix_space


				def 		__magic_name__(       self			:Union[str, Any]			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Optional[str] = None	)		->			Tuple[str]:
										__SCREAMING_SNAKE_CASE						:     List[str]					=		self._tokenizer.model.save(lowerCAmelCase__			,			name=lowerCAmelCase__	)
										return tuple(lowerCAmelCase__	)



				def 		__magic_name__(       self			:Optional[Any]			,			lowerCAmelCase__			:"Conversation"	)		->			List[int]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		[]
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase__			,			add_special_tokens=lowerCAmelCase__	) + [self.eos_token_id]	)

										if len(lowerCAmelCase__	) > self.model_max_length:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		input_ids[-self.model_max_length :]
										return input_ids
