import inspect
import unittest

from transformers import EfficientNetConfig
from transformers.testing_utils import is_pipeline_test, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import EfficientNetForImageClassification, EfficientNetModel
 from transformers.models.efficientnet.modeling_efficientnet import EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor




class  __lowerCAmelCase	:



 '''simple docstring'''

 def __init__(      self							:		Tuple							,      lowerCamelCase							:		List[str]							,      lowerCamelCase							:		List[Any]=1_3							,      lowerCamelCase							:		List[str]=3_2							,      lowerCamelCase							:		Dict=3							,      lowerCamelCase							:		Tuple=[3, 3, 5]							,      lowerCamelCase							:		Optional[int]=[3_2, 1_6, 2_4]							,      lowerCamelCase							:		Optional[Any]=[1_6, 2_4, 4_0]							,      lowerCamelCase							:		List[str]=[1, 1, 2]							,      lowerCamelCase							:		str=[1, 1, 2]							,      lowerCamelCase							:		List[Any]=[1, 6, 6]							,      lowerCamelCase							:		int=True							,      lowerCamelCase							:		Optional[Any]=True							,      lowerCamelCase							:		Union[str, Any]=3_7							,      lowerCamelCase							:		List[str]="gelu"							,      lowerCamelCase							:		List[Any]=1_0							,      )			->				Optional[int]:
        _UpperCamelCase  :							str	    =	parent
        _UpperCamelCase  :							List[Any]	    =	batch_size
        _UpperCamelCase  :							Tuple	    =	image_size
        _UpperCamelCase  :							Optional[Any]	    =	num_channels
        _UpperCamelCase  :							Tuple	    =	kernel_sizes
        _UpperCamelCase  :							Union[str, Any]	    =	in_channels
        _UpperCamelCase  :							str	    =	out_channels
        _UpperCamelCase  :							Union[str, Any]	    =	strides
        _UpperCamelCase  :							Tuple	    =	num_block_repeats
        _UpperCamelCase  :							Dict	    =	expand_ratios
        _UpperCamelCase  :							Optional[int]	    =	is_training
        _UpperCamelCase  :							int	    =	hidden_act
        _UpperCamelCase  :							str	    =	num_labels
        _UpperCamelCase  :							List[Any]	    =	use_labels

 def        __snake_case       (      self							:		Dict	)			->				str:
        _UpperCamelCase  :							Any	    =	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

        _UpperCamelCase  :							Tuple	    =	None
        if self.use_labels:
               _UpperCamelCase  :							Any	    =	ids_tensor([self.batch_size]							,      self.num_labels	)

        _UpperCamelCase  :							int	    =	self.get_config()
        return config, pixel_values, labels

 def        __snake_case       (      self							:		int	)			->				Tuple:
        return EfficientNetConfig(
            num_channels=self.num_channels							,      kernel_sizes=self.kernel_sizes							,      in_channels=self.in_channels							,      out_channels=self.out_channels							,      strides=self.strides							,      num_block_repeats=self.num_block_repeats							,      expand_ratios=self.expand_ratios							,      hidden_act=self.hidden_act							,      num_labels=self.num_labels							,      )

 def        __snake_case       (      self							:		List[Any]							,      lowerCamelCase							:		int							,      lowerCamelCase							:		str							,      lowerCamelCase							:		Optional[Any]	)			->				Union[str, Any]:
        _UpperCamelCase  :							Dict	    =	EfficientNetModel(config=lowerCamelCase	)
        model.to(lowerCamelCase	)
        model.eval()
        _UpperCamelCase  :							Union[str, Any]	    =	model(lowerCamelCase	)
        # expected last hidden states: B, C, H // 4, W // 4
        self.parent.assertEqual(
            result.last_hidden_state.shape							,      (self.batch_size, config.hidden_dim, self.image_size // 4, self.image_size // 4)							,      )

 def        __snake_case       (      self							:		int							,      lowerCamelCase							:		Dict							,      lowerCamelCase							:		str							,      lowerCamelCase							:		Any	)			->				int:
        _UpperCamelCase  :							Any	    =	EfficientNetForImageClassification(lowerCamelCase	)
        model.to(lowerCamelCase	)
        model.eval()
        _UpperCamelCase  :							List[Any]	    =	model(lowerCamelCase							,      labels=lowerCamelCase	)
        self.parent.assertEqual(result.logits.shape							,      (self.batch_size, self.num_labels)	)

 def        __snake_case       (      self							:		Dict	)			->				int:
        _UpperCamelCase  :							Optional[Any]	    =	self.prepare_config_and_inputs()
        _UpperCamelCase    ,   _UpperCamelCase    ,   _UpperCamelCase  :							List[str]	    =	config_and_inputs
        _UpperCamelCase  :							int	    =	{'pixel_values': pixel_values}
        return config, inputs_dict







@require_torch
class  __lowerCAmelCase	(							a_,						a_,						unittest.TestCase    ):



 '''simple docstring'''
 _lowerCamelCase    :						int					     =			(EfficientNetModel, EfficientNetForImageClassification) if is_torch_available() else ()
 _lowerCamelCase    :						Dict					     =			(
     {'feature-extraction': EfficientNetModel, 'image-classification': EfficientNetForImageClassification}
     if is_torch_available()
     else {}
 )

 _lowerCamelCase    :						Any					     =			False
 _lowerCamelCase    :						List[str]					     =			False
 _lowerCamelCase    :						int					     =			False
 _lowerCamelCase    :						Optional[int]					     =			False
 _lowerCamelCase    :						List[Any]					     =			False

 def        __snake_case       (      self							:		Optional[int]	)			->				Optional[int]:
        _UpperCamelCase  :							Optional[int]	    =	EfficientNetModelTester(self	)
        _UpperCamelCase  :							Any	    =	ConfigTester(
            self							,      config_class=lowerCamelCase							,      has_text_modality=lowerCamelCase							,      hidden_size=3_7	)

 def        __snake_case       (      self							:		Union[str, Any]	)			->				Optional[Any]:
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()

 def        __snake_case       (      self							:		Any	)			->				Tuple:
        return

 @unittest.skip(reason='EfficientNet does not use inputs_embeds'	)
 def        __snake_case       (      self							:		List[Any]	)			->				Tuple:
        pass

 @unittest.skip(reason='EfficientNet does not support input and output embeddings'	)
 def        __snake_case       (      self							:		Tuple	)			->				Union[str, Any]:
        pass

 @unittest.skip(reason='EfficientNet does not use feedforward chunking'	)
 def        __snake_case       (      self							:		List[str]	)			->				int:
        pass

 def        __snake_case       (      self							:		List[str]	)			->				List[str]:
        _UpperCamelCase    ,   _UpperCamelCase  :							Optional[Any]	    =	self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               _UpperCamelCase  :							str	    =	model_class(lowerCamelCase	)
               _UpperCamelCase  :							List[str]	    =	inspect.signature(model.forward	)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               _UpperCamelCase  :							str	    =	[*signature.parameters.keys()]

               _UpperCamelCase  :							Any	    =	['pixel_values']
               self.assertListEqual(arg_names[:1]							,      lowerCamelCase	)

 def        __snake_case       (      self							:		Optional[Any]	)			->				List[str]:
        _UpperCamelCase  :							Optional[int]	    =	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCamelCase	)

 def        __snake_case       (      self							:		int	)			->				Dict:
        def check_hidden_states_output(lowerCamelCase							:		str							,      lowerCamelCase							:		Tuple							,      lowerCamelCase							:		Optional[int]	):
               _UpperCamelCase  :							Optional[Any]	    =	model_class(lowerCamelCase	)
               model.to(lowerCamelCase	)
               model.eval()

               with torch.no_grad():
                      _UpperCamelCase  :							str	    =	model(**self._prepare_for_class(lowerCamelCase							,      lowerCamelCase	)	)

               _UpperCamelCase  :							Any	    =	outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states
               _UpperCamelCase  :							Any	    =	sum(config.num_block_repeats	) * 4
               self.assertEqual(len(lowerCamelCase	)							,      lowerCamelCase	)

               # EfficientNet's feature maps are of shape (batch_size, num_channels, height, width)
               self.assertListEqual(
                   list(hidden_states[0].shape[-2:]	)							,      [self.model_tester.image_size // 2, self.model_tester.image_size // 2]							,      )

        _UpperCamelCase    ,   _UpperCamelCase  :							Tuple	    =	self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               _UpperCamelCase  :							Optional[int]	    =	True
               check_hidden_states_output(lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase	)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               _UpperCamelCase  :							Tuple	    =	True

               check_hidden_states_output(lowerCamelCase							,      lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		str	)			->				int:
        _UpperCamelCase  :							Optional[Any]	    =	self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*lowerCamelCase	)

 @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'	)
 def        __snake_case       (      self							:		List[Any]	)			->				str:
        pass

 @slow
 def        __snake_case       (      self							:		List[str]	)			->				int:
        for model_name in EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _UpperCamelCase  :							Union[str, Any]	    =	EfficientNetModel.from_pretrained(lowerCamelCase	)
               self.assertIsNotNone(lowerCamelCase	)

 @is_pipeline_test
 @require_vision
 @slow
 def        __snake_case       (      self							:		Any	)			->				List[Any]:
        super().test_pipeline_image_classification()







def SCREAMING_SNAKE_CASE					(  ):
       _UpperCamelCase  :							Optional[Any]	    =	Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
       return image





@require_torch
@require_vision
class  __lowerCAmelCase	(							unittest.TestCase    ):



 '''simple docstring'''

 @cached_property
 def        __snake_case       (      self							:		Tuple	)			->				Union[str, Any]:
        return AutoImageProcessor.from_pretrained('google/efficientnet-b7'	) if is_vision_available() else None

 @slow
 def        __snake_case       (      self							:		List[Any]	)			->				List[Any]:
        _UpperCamelCase  :							Union[str, Any]	    =	EfficientNetForImageClassification.from_pretrained('google/efficientnet-b7'	).to(lowerCamelCase	)

        _UpperCamelCase  :							int	    =	self.default_image_processor
        _UpperCamelCase  :							Tuple	    =	prepare_img()
        _UpperCamelCase  :							int	    =	image_processor(images=lowerCamelCase							,      return_tensors='pt'	).to(lowerCamelCase	)

        # forward pass
        with torch.no_grad():
               _UpperCamelCase  :							List[Any]	    =	model(**lowerCamelCase	)

        # verify the logits
        _UpperCamelCase  :							int	    =	torch.Size((1, 1_0_0_0)	)
        self.assertEqual(outputs.logits.shape							,      lowerCamelCase	)

        _UpperCamelCase  :							Any	    =	torch.tensor([0.0_0_0_1, 0.0_0_0_2, 0.0_0_0_2]	).to(lowerCamelCase	)
        self.assertTrue(torch.allclose(outputs.logits[0, :3]							,      lowerCamelCase							,      atol=1e-4	)	)







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      __snake_case				(   _UpperCAmelCase	,      unittest.TestCase):
					SCREAMING_SNAKE_CASE_		:       str								=			ElectraTokenizer
					SCREAMING_SNAKE_CASE_		:       Any								=			ElectraTokenizerFast
					SCREAMING_SNAKE_CASE_		:       Tuple								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			filter_non_english
					def 	a			(						self       :						int  )							->					Optional[int]:



											'''simple docstring'''





											super().setUp()

											__magic_name__					=      [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											__magic_name__					=      os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file	,      'w'	,      encoding='utf-8'  ) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						Dict  )							->					List[Any]:



											'''simple docstring'''





											__magic_name__					=      'UNwant\u00E9d,running'
											__magic_name__					=      'unwanted, running'
											return input_text, output_text
					def 	a			(						self       :						Union[str, Any]  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class(self.vocab_file  )

											__magic_name__					=      tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(_lowerCAmelCase	,      ['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )	,      [9, 6, 7, 1_2, 1_0, 1_1]  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											if not self.test_rust_tokenizer:
																	return

											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											# With lower casing
											__magic_name__					=      self.get_tokenizer(do_lower_case=_lowerCAmelCase  )
											__magic_name__					=      self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase  )

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
					def 	a			(						self       :						str  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )	,      ['ah', '\u535A', '\u63A8', 'zz']  )
					def 	a			(						self       :						Optional[int]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['h\u00E9llo']  )
					def 	a			(						self       :						Any  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						List[str]  )							->					str:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						Union[str, Any]  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						int  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Dict  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Tuple  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
					def 	a			(						self       :						Dict  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											__magic_name__					=      {}
											for i, token in enumerate(_lowerCAmelCase  ):
																	__magic_name__					=      i
											__magic_name__					=      WordpieceTokenizer(vocab=_lowerCAmelCase	,      unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )	,      []  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )	,      ['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )	,      ['[UNK]', 'runn', '##ing']  )
					def 	a			(						self       :						List[str]  )							->					Union[str, Any]:



											'''simple docstring'''





											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
					def 	a			(						self       :						List[Any]  )							->					List[str]:



											'''simple docstring'''





											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
					def 	a			(						self       :						Optional[Any]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )
					@slow
					def 	a			(						self       :						str  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											__magic_name__					=      tokenizer.encode('sequence builders'	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      tokenizer.encode('multi-sequence build'	,      add_special_tokens=_lowerCAmelCase  )

											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  )
											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase	,      _lowerCAmelCase  )

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																							__magic_name__					=      tokenizer_r.encode_plus(
																							    _lowerCAmelCase	,      return_attention_mask=_lowerCAmelCase	,      return_token_type_ids=_lowerCAmelCase	,      return_offsets_mapping=_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase	,      )

																							__magic_name__					=      tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase	,      'do_lower_case'  ) else False
																							__magic_name__					=      (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'Allen'),
																							        ((2_1, 2_3), '##NL'),
																							        ((2_3, 2_4), '##P'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'allen'),
																							        ((2_1, 2_3), '##nl'),
																							        ((2_3, 2_4), '##p'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]	,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																							self.assertEqual([e[0] for e in expected_results]	,      tokens['offset_mapping']  )







					def 	a			(						self       :						Any  )							->					int:



											'''simple docstring'''





											__magic_name__					=      ['的', '人', '有']
											__magic_name__					=      ''.join(_lowerCAmelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      True
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

																							__magic_name__					=      False
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that only the first Chinese character is not preceded by "##".
																							__magic_name__					=      [
																							    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase  )
																							]
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

