


import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
					import torch

					from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
					import pyanvml.pyanvml as nvml


UpperCamelCase			:					Tuple         =							logging.get_logger(__name__)

class      __snake_case				(   _UpperCAmelCase):
					SCREAMING_SNAKE_CASE_		:       PyTorchBenchmarkArguments
					SCREAMING_SNAKE_CASE_		:       PretrainedConfig
					SCREAMING_SNAKE_CASE_		:       str								=			"PyTorch"
					@property
					def 	a			(						self       :						Tuple  )							->					List[str]:



											'''simple docstring'''





											return torch.__version__
					def 	a			(						self       :						Any	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					float:



											'''simple docstring'''





											__magic_name__					=      self._prepare_inference_func(_lowerCAmelCase	,      _lowerCAmelCase	,      _lowerCAmelCase  )
											return self._measure_speed(_inference  )
					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					[Memory, Optional[MemorySummary]]:



											'''simple docstring'''





											__magic_name__					=      self._prepare_inference_func(_lowerCAmelCase	,      _lowerCAmelCase	,      _lowerCAmelCase  )
											return self._measure_memory(_inference  )
					def 	a			(						self       :						str	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					float:



											'''simple docstring'''





											__magic_name__					=      self._prepare_train_func(_lowerCAmelCase	,      _lowerCAmelCase	,      _lowerCAmelCase  )
											return self._measure_speed(_train  )
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					[Memory, Optional[MemorySummary]]:



											'''simple docstring'''





											__magic_name__					=      self._prepare_train_func(_lowerCAmelCase	,      _lowerCAmelCase	,      _lowerCAmelCase  )
											return self._measure_memory(_train  )
					def 	a			(						self       :						Optional[int]	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					Callable[[], None]:



											'''simple docstring'''





											__magic_name__					=      self.config_dict[model_name]

											if self.args.torchscript:
																	__magic_name__					=      True

											__magic_name__					=      (
											    hasattr(_lowerCAmelCase	,      'architectures'  )
											    and isinstance(config.architectures	,      _lowerCAmelCase  )
											    and len(config.architectures  ) > 0
											)
											if not self.args.only_pretrain_model and has_model_class_in_config:
																	try:
																							__magic_name__					=      config.architectures[0]
																							__magic_name__					=      __import__('transformers'	,      fromlist=[model_class]  )
																							__magic_name__					=      getattr(_lowerCAmelCase	,      _lowerCAmelCase  )
																							__magic_name__					=      model_cls(_lowerCAmelCase  )
																	except ImportError:
																							raise ImportError(
																							    F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																							    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'  )
											else:
																	__magic_name__					=      MODEL_MAPPING[config.__class__](_lowerCAmelCase  )

											model.eval()
											model.to(self.args.device  )

											# encoder-decoder has vocab size saved differently
											__magic_name__					=      config.vocab_size if hasattr(_lowerCAmelCase	,      'vocab_size'  ) else config.encoder.vocab_size
											__magic_name__					=      torch.randint(_lowerCAmelCase	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device  )

											if self.args.fpaa:
																	logger.info('Running training in Mixed Precision...'  )
																	if not self.args.is_gpu:
																							raise ValueError('Mixed precision is possible only for GPU.'  )
																	# amp seems to have memory leaks so that memory usage
																	# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
																	model.half()

											if self.args.torchscript:
																	with torch.no_grad():
																							__magic_name__					=      torch.jit.trace(_lowerCAmelCase	,      _lowerCAmelCase  )
											else:
																	__magic_name__					=      model

											def encoder_decoder_forward():
																	with torch.no_grad():
																							__magic_name__					=      inference_model(_lowerCAmelCase	,      decoder_input_ids=_lowerCAmelCase  )
																	return outputs

											def encoder_forward():
																	with torch.no_grad():
																							__magic_name__					=      inference_model(_lowerCAmelCase  )
																	return outputs

											__magic_name__					=      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
											return _forward
					def 	a			(						self       :						Any	,      _lowerCAmelCase       :						str	,      _lowerCAmelCase       :						int	,      _lowerCAmelCase       :						int  )							->					Callable[[], None]:



											'''simple docstring'''





											__magic_name__					=      self.config_dict[model_name]

											__magic_name__					=      (
											    hasattr(_lowerCAmelCase	,      'architectures'  )
											    and isinstance(config.architectures	,      _lowerCAmelCase  )
											    and len(config.architectures  ) > 0
											)
											if not self.args.only_pretrain_model and has_model_class_in_config:
																	try:
																							__magic_name__					=      config.architectures[0]
																							__magic_name__					=      __import__('transformers'	,      fromlist=[model_class]  )
																							__magic_name__					=      getattr(_lowerCAmelCase	,      _lowerCAmelCase  )
																							__magic_name__					=      model_cls(_lowerCAmelCase  )
																	except ImportError:
																							raise ImportError(
																							    F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																							    ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'  )
											else:
																	__magic_name__					=      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_lowerCAmelCase  )

											if self.args.torchscript:
																	raise NotImplementedError('Training for torchscript is currently not implemented'  )
											else:
																	__magic_name__					=      model

											model.train()
											model.to(self.args.device  )

											# encoder-decoder has vocab size saved differently
											__magic_name__					=      config.vocab_size if hasattr(_lowerCAmelCase	,      'vocab_size'  ) else config.encoder.vocab_size
											__magic_name__					=      torch.randint(_lowerCAmelCase	,      (batch_size, sequence_length)	,      dtype=torch.long	,      device=self.args.device  )

											if self.args.fpaa:
																	logger.info('Running training in Mixed Precision...'  )
																	if not self.args.is_gpu:
																							raise ValueError('Mixed precision is possible only for GPU.'  )

																	# amp seems to have memory leaks so that memory usage
																	# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
																	model.half()

											def compute_loss_and_backprob_encoder():
																	__magic_name__					=      train_model(_lowerCAmelCase	,      labels=_lowerCAmelCase  )[0]
																	loss.backward()
																	return loss

											def compute_loss_and_backprob_encoder_decoder():
																	__magic_name__					=      train_model(_lowerCAmelCase	,      decoder_input_ids=_lowerCAmelCase	,      labels=_lowerCAmelCase  )[0]
																	loss.backward()
																	return loss

											__magic_name__					=      (
											    compute_loss_and_backprob_encoder_decoder
											    if config.is_encoder_decoder
											    else compute_loss_and_backprob_encoder
											)
											return _train
					def 	a			(						self       :						List[str]	,      _lowerCAmelCase       :						Optional[int]  )							->					float:



											'''simple docstring'''





											try:
																	if self.args.is_tpu or self.args.torchscript:
																							# run additional 10 times to stabilize compilation for tpu and torchscript
																							logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'  )
																							timeit.repeat(
																							    _lowerCAmelCase	,      repeat=1	,      number=5	,      )

																	# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																	__magic_name__					=      timeit.repeat(
																	    _lowerCAmelCase	,      repeat=self.args.repeat	,      number=1_0	,      )

																	if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
																							import torch_xla.debug.metrics as met

																							self.print_fn(met.metrics_report()  )

																	return min(_lowerCAmelCase  ) / 10.0
											except RuntimeError as e:
																	self.print_fn(F'''Doesn\'t fit on GPU. {e}'''  )
																	return "N/A"







					def 	a			(						self       :						str	,      _lowerCAmelCase       :						Callable[[], None]  )							->					[Memory, MemorySummary]:



											'''simple docstring'''





											try:
																	if self.args.trace_memory_line_by_line:
																							__magic_name__					=      start_memory_tracing('transformers'  )

																	if self.args.is_tpu:
																							# tpu
																							raise NotImplementedError(
																							    'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
																							    ' `--no-memory` or `args.memory=False`'  )
																	elif self.args.is_gpu:
																							if not is_pyanvml_available():
																													logger.warning(
																													    'py3nvml not installed, we won\'t log GPU memory usage. '
																													    'Install py3nvml (pip install py3nvml) to log information about GPU.'  )
																													__magic_name__					=      'N/A'
																							else:
																													logger.info(
																													    'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
																													    ' on the same GPU.'  )
																													# init nvml
																													nvml.nvmlInit()
																													func()
																													__magic_name__					=      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx  )
																													__magic_name__					=      nvml.nvmlDeviceGetMemoryInfo(_lowerCAmelCase  )
																													__magic_name__					=      meminfo.used
																													__magic_name__					=      Memory(_lowerCAmelCase  )
																													# shutdown nvml
																													nvml.nvmlShutdown()
																	else:
																							# cpu
																							__magic_name__					=      measure_peak_memory_cpu(_lowerCAmelCase  )
																							__magic_name__					=      Memory(_lowerCAmelCase  ) if isinstance(_lowerCAmelCase	,      _lowerCAmelCase  ) else memory_bytes

																	if self.args.trace_memory_line_by_line:
																							__magic_name__					=      stop_memory_tracing(_lowerCAmelCase  )
																	else:
																							__magic_name__					=      None

																	return memory, summary
											except RuntimeError as e:
																	self.print_fn(F'''Doesn\'t fit on GPU. {e}'''  )
																	return "N/A", None





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      __snake_case				(   _UpperCAmelCase	,      unittest.TestCase):
					SCREAMING_SNAKE_CASE_		:       str								=			ElectraTokenizer
					SCREAMING_SNAKE_CASE_		:       Any								=			ElectraTokenizerFast
					SCREAMING_SNAKE_CASE_		:       Tuple								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			True
					SCREAMING_SNAKE_CASE_		:       str								=			filter_non_english
					def 	a			(						self       :						int  )							->					Optional[int]:



											'''simple docstring'''





											super().setUp()

											__magic_name__					=      [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											__magic_name__					=      os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES['vocab_file']  )
											with open(self.vocab_file	,      'w'	,      encoding='utf-8'  ) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
					def 	a			(						self       :						Tuple	,      _lowerCAmelCase       :						Dict  )							->					List[Any]:



											'''simple docstring'''





											__magic_name__					=      'UNwant\u00E9d,running'
											__magic_name__					=      'unwanted, running'
											return input_text, output_text
					def 	a			(						self       :						Union[str, Any]  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class(self.vocab_file  )

											__magic_name__					=      tokenizer.tokenize('UNwant\u00E9d,running'  )
											self.assertListEqual(_lowerCAmelCase	,      ['un', '##want', '##ed', ',', 'runn', '##ing']  )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase  )	,      [9, 6, 7, 1_2, 1_0, 1_1]  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											if not self.test_rust_tokenizer:
																	return

											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											# With lower casing
											__magic_name__					=      self.get_tokenizer(do_lower_case=_lowerCAmelCase  )
											__magic_name__					=      self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase  )

											__magic_name__					=      'UNwant\u00E9d,running'

											__magic_name__					=      tokenizer.tokenize(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.tokenize(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

											__magic_name__					=      self.get_rust_tokenizer()
											__magic_name__					=      tokenizer.encode(_lowerCAmelCase  )
											__magic_name__					=      rust_tokenizer.encode(_lowerCAmelCase  )
											self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
					def 	a			(						self       :						str  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )	,      ['ah', '\u535A', '\u63A8', 'zz']  )
					def 	a			(						self       :						Optional[int]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['hello', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hällo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['h\u00E9llo']  )
					def 	a			(						self       :						Any  )							->					Tuple:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						List[str]  )							->					str:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['hallo', '!', 'how', 'are', 'you', '?']  )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )	,      ['hello']  )
					def 	a			(						self       :						Union[str, Any]  )							->					Optional[Any]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						int  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Dict  )							->					int:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      strip_accents=_lowerCAmelCase  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )	,      ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
					def 	a			(						self       :						Tuple  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      BasicTokenizer(do_lower_case=_lowerCAmelCase	,      never_split=['[UNK]']  )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )	,      ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
					def 	a			(						self       :						Dict  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											__magic_name__					=      {}
											for i, token in enumerate(_lowerCAmelCase  ):
																	__magic_name__					=      i
											__magic_name__					=      WordpieceTokenizer(vocab=_lowerCAmelCase	,      unk_token='[UNK]'  )

											self.assertListEqual(tokenizer.tokenize(''  )	,      []  )

											self.assertListEqual(tokenizer.tokenize('unwanted running'  )	,      ['un', '##want', '##ed', 'runn', '##ing']  )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'  )	,      ['[UNK]', 'runn', '##ing']  )
					def 	a			(						self       :						List[str]  )							->					Union[str, Any]:



											'''simple docstring'''





											self.assertTrue(_is_whitespace(' '  )  )
											self.assertTrue(_is_whitespace('\t'  )  )
											self.assertTrue(_is_whitespace('\r'  )  )
											self.assertTrue(_is_whitespace('\n'  )  )
											self.assertTrue(_is_whitespace('\u00A0'  )  )

											self.assertFalse(_is_whitespace('A'  )  )
											self.assertFalse(_is_whitespace('-'  )  )
					def 	a			(						self       :						Optional[int]  )							->					Dict:



											'''simple docstring'''





											self.assertTrue(_is_control('\u0005'  )  )

											self.assertFalse(_is_control('A'  )  )
											self.assertFalse(_is_control(' '  )  )
											self.assertFalse(_is_control('\t'  )  )
											self.assertFalse(_is_control('\r'  )  )
					def 	a			(						self       :						List[Any]  )							->					List[str]:



											'''simple docstring'''





											self.assertTrue(_is_punctuation('-'  )  )
											self.assertTrue(_is_punctuation('$'  )  )
											self.assertTrue(_is_punctuation('`'  )  )
											self.assertTrue(_is_punctuation('.'  )  )

											self.assertFalse(_is_punctuation('A'  )  )
											self.assertFalse(_is_punctuation(' '  )  )
					def 	a			(						self       :						Optional[Any]  )							->					int:



											'''simple docstring'''





											__magic_name__					=      self.get_tokenizer()
											__magic_name__					=      self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_lowerCAmelCase  ) for t in ['Test', '\xad', 'test']]	,      [['[UNK]'], [], ['[UNK]']]  )
					@slow
					def 	a			(						self       :						str  )							->					List[str]:



											'''simple docstring'''





											__magic_name__					=      self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

											__magic_name__					=      tokenizer.encode('sequence builders'	,      add_special_tokens=_lowerCAmelCase  )
											__magic_name__					=      tokenizer.encode('multi-sequence build'	,      add_special_tokens=_lowerCAmelCase  )

											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  )
											__magic_name__					=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase	,      _lowerCAmelCase  )

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
					def 	a			(						self       :						int  )							->					Union[str, Any]:



											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																							__magic_name__					=      tokenizer_r.encode_plus(
																							    _lowerCAmelCase	,      return_attention_mask=_lowerCAmelCase	,      return_token_type_ids=_lowerCAmelCase	,      return_offsets_mapping=_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase	,      )

																							__magic_name__					=      tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase	,      'do_lower_case'  ) else False
																							__magic_name__					=      (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'Allen'),
																							        ((2_1, 2_3), '##NL'),
																							        ((2_3, 2_4), '##P'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'allen'),
																							        ((2_1, 2_3), '##nl'),
																							        ((2_3, 2_4), '##p'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]	,      tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																							self.assertEqual([e[0] for e in expected_results]	,      tokens['offset_mapping']  )







					def 	a			(						self       :						Any  )							->					int:



											'''simple docstring'''





											__magic_name__					=      ['的', '人', '有']
											__magic_name__					=      ''.join(_lowerCAmelCase  )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																							__magic_name__					=      True
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

																							__magic_name__					=      False
																							__magic_name__					=      self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )
																							__magic_name__					=      self.tokenizer_class.from_pretrained(_lowerCAmelCase	,      **_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.encode(_lowerCAmelCase	,      add_special_tokens=_lowerCAmelCase  )

																							__magic_name__					=      tokenizer_r.convert_ids_to_tokens(_lowerCAmelCase  )
																							__magic_name__					=      tokenizer_p.convert_ids_to_tokens(_lowerCAmelCase  )

																							# it is expected that only the first Chinese character is not preceded by "##".
																							__magic_name__					=      [
																							    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCAmelCase  )
																							]
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )
																							self.assertListEqual(_lowerCAmelCase	,      _lowerCAmelCase  )

