


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast


class        _a							(			unittest.TestCase			):

							def       lowerCamelCase_ (							self:	List[Any]							)			->   int:





									"""simple docstring"""







									lowercase__            =      mock.Mock()
									lowercase__            =      500
									lowercase__            =      {}
									lowercase__            =      HTTPError
									lowercase__            =      {}

									# Download this model to make sure it's in the cache.
									lowercase__            =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''    ,						return_value=UpperCamelCase_							) as mock_head:
											lowercase__            =      BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)
											# This check we did call the fake head request
											mock_head.assert_called()

							@require_tokenizers
							def       lowerCamelCase_ (							self:	int							)			->   List[Any]:





									"""simple docstring"""







									lowercase__            =      mock.Mock()
									lowercase__            =      500
									lowercase__            =      {}
									lowercase__            =      HTTPError
									lowercase__            =      {}

									# Download this model to make sure it's in the cache.
									lowercase__            =      GPTaTokenizerFast.from_pretrained('''gpt2'''							)

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''    ,						return_value=UpperCamelCase_							) as mock_head:
											lowercase__            =      GPTaTokenizerFast.from_pretrained('''gpt2'''							)
											# This check we did call the fake head request
											mock_head.assert_called()

							def       lowerCamelCase_ (							self:	List[str]							)			->   Union[str, Any]:





									"""simple docstring"""







									try:
											lowercase__            =      tempfile.mktemp()
											with open(UpperCamelCase_    ,						'''wb'''							) as f:
													http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''    ,						UpperCamelCase_							)

											lowercase__            =      AlbertTokenizer.from_pretrained(UpperCamelCase_							)
									finally:
											os.remove(UpperCamelCase_							)

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile('''tokenizer.json'''							):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
									try:
											with open('''tokenizer.json'''    ,						'''wb'''							) as f:
													http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''    ,						UpperCamelCase_							)
											lowercase__            =      AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''							)
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size    ,						1_000							)
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
											os.remove('''tokenizer.json'''							)


							def       lowerCamelCase_ (							self:	str							)			->   Optional[int]:





									"""simple docstring"""







									lowercase__            =      AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''							)


@is_staging_test
class        _a							(			unittest.TestCase			):
							_lowercase	:			Optional[Any]        = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']

							@classmethod
							def       lowerCamelCase_ (							cls:	Optional[int]							)			->   str:





									"""simple docstring"""







									lowercase__            =      TOKEN
									HfFolder.save_token(UpperCamelCase_							)

							@classmethod
							def       lowerCamelCase_ (							cls:	Tuple							)			->   Any:





									"""simple docstring"""







									try:
											delete_repo(token=cls._token    ,						repo_id='''test-tokenizer'''							)
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token    ,						repo_id='''valid_org/test-tokenizer-org'''							)
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token    ,						repo_id='''test-dynamic-tokenizer'''							)
									except HTTPError:
											pass

							def       lowerCamelCase_ (							self:	Dict							)			->   str:





									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											lowercase__            =      os.path.join(UpperCamelCase_    ,						'''vocab.txt'''							)
											with open(UpperCamelCase_    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]							)							)
											lowercase__            =      BertTokenizer(UpperCamelCase_							)

									tokenizer.push_to_hub('''test-tokenizer'''    ,						use_auth_token=self._token							)
									lowercase__            =      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'							)
									self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab							)

									# Reset repo
									delete_repo(token=self._token    ,						repo_id='''test-tokenizer'''							)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(UpperCamelCase_    ,						repo_id='''test-tokenizer'''    ,						push_to_hub=UpperCamelCase_    ,						use_auth_token=self._token							)

									lowercase__            =      BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'							)
									self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab							)

							def       lowerCamelCase_ (							self:	Optional[int]							)			->   Optional[Any]:





									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											lowercase__            =      os.path.join(UpperCamelCase_    ,						'''vocab.txt'''							)
											with open(UpperCamelCase_    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]							)							)
											lowercase__            =      BertTokenizer(UpperCamelCase_							)

									tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''    ,						use_auth_token=self._token							)
									lowercase__            =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''							)
									self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab							)

									# Reset repo
									delete_repo(token=self._token    ,						repo_id='''valid_org/test-tokenizer-org'''							)

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    UpperCamelCase_    ,						repo_id='''valid_org/test-tokenizer-org'''    ,						push_to_hub=UpperCamelCase_    ,						use_auth_token=self._token							)

									lowercase__            =      BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''							)
									self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab							)


							@require_tokenizers
							def       lowerCamelCase_ (							self:	Any							)			->   Tuple:





									"""simple docstring"""







									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											lowercase__            =      os.path.join(UpperCamelCase_    ,						'''vocab.txt'''							)
											with open(UpperCamelCase_    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]							)							)
											lowercase__            =      CustomTokenizer(UpperCamelCase_							)

									# No fast custom tokenizer
									tokenizer.push_to_hub('''test-dynamic-tokenizer'''    ,						use_auth_token=self._token							)

									lowercase__            =      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'    ,						trust_remote_code=UpperCamelCase_							)
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizer'''							)

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											lowercase__            =      os.path.join(UpperCamelCase_    ,						'''vocab.txt'''							)
											with open(UpperCamelCase_    ,						'''w'''    ,						encoding='''utf-8'''							) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]							)							)

											lowercase__            =      BertTokenizerFast.from_pretrained(UpperCamelCase_							)
											bert_tokenizer.save_pretrained(UpperCamelCase_							)
											lowercase__            =      CustomTokenizerFast.from_pretrained(UpperCamelCase_							)

									tokenizer.push_to_hub('''test-dynamic-tokenizer'''    ,						use_auth_token=self._token							)

									lowercase__            =      AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'    ,						trust_remote_code=UpperCamelCase_							)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizerFast'''							)
									lowercase__            =      AutoTokenizer.from_pretrained(
									    f'{USER}/test-dynamic-tokenizer'    ,						use_fast=UpperCamelCase_    ,						trust_remote_code=UpperCamelCase_							)
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizer'''							)


class        _a							(			unittest.TestCase			):

							def       lowerCamelCase_ (							self:	Tuple							)			->   Any:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''Hello 友達'''							)
									self.assertEqual(trie.data    ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}							)
									trie.add('''Hello'''							)
									trie.data
									self.assertEqual(trie.data    ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}							)

							def       lowerCamelCase_ (							self:	Dict							)			->   int:





									"""simple docstring"""







									lowercase__            =      Trie()
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''							)    ,						['''[CLS] This is a extra_id_100''']							)
									trie.add('''[CLS]'''							)
									trie.add('''extra_id_1'''							)
									trie.add('''extra_id_100'''							)
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''							)    ,						['''[CLS]''', ''' This is a ''', '''extra_id_100''']							)

							def       lowerCamelCase_ (							self:	Optional[int]							)			->   List[str]:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''A'''							)
									self.assertEqual(trie.split('''ABC'''							)    ,						['''A''', '''BC''']							)
									self.assertEqual(trie.split('''BCA'''							)    ,						['''BC''', '''A''']							)

							def       lowerCamelCase_ (							self:	str							)			->   Optional[Any]:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''TOKEN]'''							)
									trie.add('''[SPECIAL_TOKEN]'''							)
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''							)    ,						['''This is something ''', '''[SPECIAL_TOKEN]''']							)

							def       lowerCamelCase_ (							self:	List[str]							)			->   Dict:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''A'''							)
									trie.add('''P'''							)
									trie.add('''[SPECIAL_TOKEN]'''							)
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''							)    ,						['''This is something ''', '''[SPECIAL_TOKEN]''']							)

							def       lowerCamelCase_ (							self:	Dict							)			->   Dict:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''AB'''							)
									trie.add('''B'''							)
									trie.add('''C'''							)
									self.assertEqual(trie.split('''ABC'''							)    ,						['''AB''', '''C''']							)

							def       lowerCamelCase_ (							self:	Union[str, Any]							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      Trie()
									trie.add('''ABC'''							)
									trie.add('''B'''							)
									trie.add('''CD'''							)
									self.assertEqual(trie.split('''ABCD'''							)    ,						['''ABC''', '''D''']							)


							def       lowerCamelCase_ (							self:	str							)			->   Union[str, Any]:





									"""simple docstring"""







									lowercase__            =      Trie()
									lowercase__            =      trie.cut_text('''ABC'''    ,						[0, 0, 2, 1, 2, 3]							)
									self.assertEqual(UpperCamelCase_    ,						['''AB''', '''C''']							)









'''simple docstring'''




from collections import OrderedDict
from typing import Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...feature_extraction_utils import FeatureExtractionMixin
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import TensorType, logging


__SCREAMING_SNAKE_CASE     :Dict       =				logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE     :int       =				{
    '''deepmind/language-perceiver''': '''https://huggingface.co/deepmind/language-perceiver/resolve/main/config.json''',
    # See all Perceiver models at https://huggingface.co/models?filter=perceiver
}







class   A_	(  lowerCAmelCase_			):
 _lowerCamelCase  : int           =							"""perceiver"""




 def __init__(					self       :    Any	,			snake_case_       :    List[Any]=2_5_6	,			snake_case_       :    str=1_2_8_0	,			snake_case_       :    Optional[int]=7_6_8	,			snake_case_       :    int=1	,			snake_case_       :    List[Any]=2_6	,			snake_case_       :    Dict=8	,			snake_case_       :    List[Any]=8	,			snake_case_       :    Tuple=None	,			snake_case_       :    Tuple=None	,			snake_case_       :    Any="kv"	,			snake_case_       :    Any=1	,			snake_case_       :    List[str]=1	,			snake_case_       :    Optional[int]="gelu"	,			snake_case_       :    List[Any]=0.1	,			snake_case_       :    Dict=0.0_2	,			snake_case_       :    int=1e-12	,			snake_case_       :    List[str]=True	,			snake_case_       :    str=2_6_2	,			snake_case_       :    Optional[Any]=2_0_4_8	,			snake_case_       :    Union[str, Any]=5_6	,			snake_case_       :    Dict=[3_6_8, 4_9_6]	,			snake_case_       :    Tuple=1_6	,			snake_case_       :    Union[str, Any]=1_9_2_0	,			snake_case_       :    List[Any]=1_6	,			snake_case_       :    Tuple=[1, 1_6, 2_2_4, 2_2_4]	,			**snake_case_       :    List[Any]	,			):
      super().__init__(**snake_case_    )

      _UpperCAmelCase     							=   num_latents
      _UpperCAmelCase     							=   d_latents
      _UpperCAmelCase     							=   d_model
      _UpperCAmelCase     							=   num_blocks
      _UpperCAmelCase     							=   num_self_attends_per_block
      _UpperCAmelCase     							=   num_self_attention_heads
      _UpperCAmelCase     							=   num_cross_attention_heads
      _UpperCAmelCase     							=   qk_channels
      _UpperCAmelCase     							=   v_channels
      _UpperCAmelCase     							=   cross_attention_shape_for_attention
      _UpperCAmelCase     							=   self_attention_widening_factor
      _UpperCAmelCase     							=   cross_attention_widening_factor
      _UpperCAmelCase     							=   hidden_act
      _UpperCAmelCase     							=   attention_probs_dropout_prob
      _UpperCAmelCase     							=   initializer_range
      _UpperCAmelCase     							=   layer_norm_eps
      _UpperCAmelCase     							=   use_query_residual
      # masked language modeling attributes
      _UpperCAmelCase     							=   vocab_size
      _UpperCAmelCase     							=   max_position_embeddings
      # image classification attributes
      _UpperCAmelCase     							=   image_size
      # flow attributes
      _UpperCAmelCase     							=   train_size
      # multimodal autoencoding attributes
      _UpperCAmelCase     							=   num_frames
      _UpperCAmelCase     							=   audio_samples_per_frame
      _UpperCAmelCase     							=   samples_per_patch
      _UpperCAmelCase     							=   output_shape







class   A_	(  lowerCAmelCase_			):




 @property
 def      lowercase	(					self       :    int    ):
      if self.task == "multiple-choice":
           _UpperCAmelCase     							=   {0: "batch", 1: "choice", 2: "sequence"}
      else:
           _UpperCAmelCase     							=   {0: "batch", 1: "sequence"}
      return OrderedDict(
          [
              ("inputs", dynamic_axis),
              ("attention_mask", dynamic_axis),
          ]    )




 @property
 def      lowercase	(					self       :    Optional[Any]    ):
      return 1e-4




 def      lowercase	(					self       :    List[str]	,			snake_case_       :    Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]	,			snake_case_       :    int = -1	,			snake_case_       :    int = -1	,			snake_case_       :    int = -1	,			snake_case_       :    bool = False	,			snake_case_       :    Optional[TensorType] = None	,			snake_case_       :    int = 3	,			snake_case_       :    int = 4_0	,			snake_case_       :    int = 4_0	,			):
      # copied from `transformers.onnx.config.OnnxConfig` and slightly altered/simplified

      if isinstance(snake_case_	,			snake_case_    ):
           # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
           _UpperCAmelCase     							=   compute_effective_axis_dimension(
               snake_case_	,			fixed_dimension=OnnxConfig.default_fixed_batch	,			num_token_to_add=0    )
           # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
           _UpperCAmelCase     							=   preprocessor.num_special_tokens_to_add(snake_case_    )
           _UpperCAmelCase     							=   compute_effective_axis_dimension(
               snake_case_	,			fixed_dimension=OnnxConfig.default_fixed_sequence	,			num_token_to_add=snake_case_    )
           # Generate dummy inputs according to compute batch and sequence
           _UpperCAmelCase     							=   [" ".join(["a"]    ) * seq_length] * batch_size
           _UpperCAmelCase     							=   dict(preprocessor(snake_case_	,			return_tensors=snake_case_    )    )
           _UpperCAmelCase     							=   inputs.pop("input_ids"    )
           return inputs
      elif isinstance(snake_case_	,			snake_case_    ) and preprocessor.model_input_names[0] == "pixel_values":
           # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
           _UpperCAmelCase     							=   compute_effective_axis_dimension(snake_case_	,			fixed_dimension=OnnxConfig.default_fixed_batch    )
           _UpperCAmelCase     							=   self._generate_dummy_images(snake_case_	,			snake_case_	,			snake_case_	,			snake_case_    )
           _UpperCAmelCase     							=   dict(preprocessor(images=snake_case_	,			return_tensors=snake_case_    )    )
           _UpperCAmelCase     							=   inputs.pop("pixel_values"    )
           return inputs
      else:
           raise ValueError(
               "Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor."    )




