





'''simple docstring'''


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    MODEL_FOR_PRETRAINING_MAPPING,
				    FunnelBaseModel,
				    FunnelForMaskedLM,
				    FunnelForMultipleChoice,
				    FunnelForPreTraining,
				    FunnelForQuestionAnswering,
				    FunnelForSequenceClassification,
				    FunnelForTokenClassification,
				    FunnelModel,
				)



class 				__UpperCAmelCase	:







		'''simple docstring'''







		def __init__(self    :   Dict   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Optional[int]=13   ,   _lowerCAmelCase    :   List[str]=7   ,   _lowerCAmelCase    :   Optional[Any]=True   ,   _lowerCAmelCase    :   Optional[int]=True   ,   _lowerCAmelCase    :   List[Any]=True   ,   _lowerCAmelCase    :   Any=True   ,   _lowerCAmelCase    :   Dict=99   ,   _lowerCAmelCase    :   int=[1, 1, 2]   ,   _lowerCAmelCase    :   List[str]=1   ,   _lowerCAmelCase    :   List[str]=32   ,   _lowerCAmelCase    :   Union[str, Any]=4   ,   _lowerCAmelCase    :   Tuple=8   ,   _lowerCAmelCase    :   List[str]=37   ,   _lowerCAmelCase    :   List[str]="gelu_new"   ,   _lowerCAmelCase    :   int=0.1   ,   _lowerCAmelCase    :   List[str]=0.1   ,   _lowerCAmelCase    :   Union[str, Any]=0.0   ,   _lowerCAmelCase    :   Optional[Any]=512   ,   _lowerCAmelCase    :   str=3   ,   _lowerCAmelCase    :   int=0.02   ,   _lowerCAmelCase    :   str=3   ,   _lowerCAmelCase    :   Optional[int]=4   ,   _lowerCAmelCase    :   Optional[Any]=None   ,   _lowerCAmelCase    :   List[Any]=False   ,   ):
			A					=   parent
			A					=   batch_size
			A					=   seq_length
			A					=   is_training
			A					=   use_input_mask
			A					=   use_token_type_ids
			A					=   use_labels
			A					=   vocab_size
			A					=   block_sizes
			A					=   num_decoder_layers
			A					=   d_model
			A					=   n_head
			A					=   d_head
			A					=   d_inner
			A					=   hidden_act
			A					=   hidden_dropout
			A					=   attention_dropout
			A					=   activation_dropout
			A					=   max_position_embeddings
			A					=   type_vocab_size
			A					=   2
			A					=   num_labels
			A					=   num_choices
			A					=   scope
			A					=   initializer_std

			# Used in the tests to check the size of the first attention layer
			A					=   n_head
			# Used in the tests to check the size of the first hidden state
			A					=   self.d_model
			# Used in the tests to check the number of output hidden states/attentions
			A					=   sum(self.block_sizes   ) + (0 if base else self.num_decoder_layers)
			# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
			# the last hidden state of the first block (which is the first hidden state of the decoder).
			if not base:
				A					=   self.num_hidden_layers + 2







		def 							A  (self    :   Optional[int]   ):
			A					=   ids_tensor([self.batch_size, self.seq_length]   ,   self.vocab_size   )

			A					=   None
			if self.use_input_mask:
				A					=   ids_tensor([self.batch_size, self.seq_length]   ,   vocab_size=2   )

			A					=   None
			if self.use_token_type_ids:
				A					=   ids_tensor([self.batch_size, self.seq_length]   ,   self.type_vocab_size   )

			A					=   None
			A					=   None
			A					=   None
			if self.use_labels:
				A					=   ids_tensor([self.batch_size]   ,   self.type_sequence_label_size   )
				A					=   ids_tensor([self.batch_size, self.seq_length]   ,   self.num_labels   )
				A					=   ids_tensor([self.batch_size]   ,   self.num_choices   )
				A					=   ids_tensor([self.batch_size, self.seq_length]   ,   1   )

			A					=   self.get_config()

			return (
			    config,
			    input_ids,
			    token_type_ids,
			    input_mask,
			    sequence_labels,
			    token_labels,
			    choice_labels,
			    fake_token_labels,
			)







		def 							A  (self    :   Any   ):
			return FunnelConfig(
			    vocab_size=self.vocab_size   ,   block_sizes=self.block_sizes   ,   num_decoder_layers=self.num_decoder_layers   ,   d_model=self.d_model   ,   n_head=self.n_head   ,   d_head=self.d_head   ,   d_inner=self.d_inner   ,   hidden_act=self.hidden_act   ,   hidden_dropout=self.hidden_dropout   ,   attention_dropout=self.attention_dropout   ,   activation_dropout=self.activation_dropout   ,   max_position_embeddings=self.max_position_embeddings   ,   type_vocab_size=self.type_vocab_size   ,   initializer_std=self.initializer_std   ,   )







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   List[str]   ,   ):
			A					=   FunnelModel(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )
			A					=   model(_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, self.seq_length, self.d_model)   )

			A					=   False
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, self.seq_length, self.d_model)   )

			A					=   False
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, self.seq_length, self.d_model)   )







		def 							A  (self    :   List[Any]   ,   _lowerCAmelCase    :   Optional[Any]   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   Union[str, Any]   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   List[str]   ,   ):
			A					=   FunnelBaseModel(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )
			A					=   model(_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, 2, self.d_model)   )

			A					=   False
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, 3, self.d_model)   )

			A					=   False
			A					=   model(_lowerCAmelCase   )
			self.parent.assertEqual(result.last_hidden_state.shape   ,   (self.batch_size, 2, self.d_model)   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   str   ,   ):
			A					=   self.num_labels
			A					=   FunnelForPreTraining(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   labels=_lowerCAmelCase   )
			self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.seq_length)   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   Optional[Any]   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Optional[Any]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   str   ,   ):
			A					=   FunnelForMaskedLM(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   labels=_lowerCAmelCase   )
			self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.seq_length, self.vocab_size)   )







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   str   ,   ):
			A					=   self.num_labels
			A					=   FunnelForSequenceClassification(_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   labels=_lowerCAmelCase   )
			self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.num_labels)   )







		def 							A  (self    :   List[Any]   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   str   ,   ):
			A					=   self.num_choices
			A					=   FunnelForMultipleChoice(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   input_ids.unsqueeze(1   ).expand(-1   ,   self.num_choices   ,   -1   ).contiguous()
			A					=   token_type_ids.unsqueeze(1   ).expand(-1   ,   self.num_choices   ,   -1   ).contiguous()
			A					=   input_mask.unsqueeze(1   ).expand(-1   ,   self.num_choices   ,   -1   ).contiguous()
			A					=   model(
			    _lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   labels=_lowerCAmelCase   ,   )
			self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.num_choices)   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   Union[str, Any]   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   Optional[Any]   ,   _lowerCAmelCase    :   Optional[Any]   ,   ):
			A					=   self.num_labels
			A					=   FunnelForTokenClassification(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   labels=_lowerCAmelCase   )
			self.parent.assertEqual(result.logits.shape   ,   (self.batch_size, self.seq_length, self.num_labels)   )







		def 							A  (self    :   Any   ,   _lowerCAmelCase    :   Tuple   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Optional[int]   ,   ):
			A					=   FunnelForQuestionAnswering(config=_lowerCAmelCase   )
			model.to(_lowerCAmelCase   )
			model.eval()
			A					=   model(
			    _lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   ,   start_positions=_lowerCAmelCase   ,   end_positions=_lowerCAmelCase   ,   )
			self.parent.assertEqual(result.start_logits.shape   ,   (self.batch_size, self.seq_length)   )
			self.parent.assertEqual(result.end_logits.shape   ,   (self.batch_size, self.seq_length)   )







		def 							A  (self    :   List[Any]   ):
			A					=   self.prepare_config_and_inputs()
			(
			    (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     (
			    A
			)	,     
			)					=   config_and_inputs
			A					=   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
			return config, inputs_dict






@require_torch
class 				__UpperCAmelCase	(							A__		,       A__		,       unittest.TestCase    ):







		'''simple docstring'''







		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     (
		    (
		        FunnelModel,
		        FunnelForMaskedLM,
		        FunnelForPreTraining,
		        FunnelForQuestionAnswering,
		        FunnelForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		__lowerCAmelCase								=     (
		    {
		        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
		        '''fill-mask''': FunnelForMaskedLM,
		        '''question-answering''': FunnelForQuestionAnswering,
		        '''text-classification''': FunnelForSequenceClassification,
		        '''token-classification''': FunnelForTokenClassification,
		        '''zero-shot''': FunnelForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   List[str]   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   List[Any]=False   ):
			A					=   super()._prepare_for_class(_lowerCAmelCase   ,   _lowerCAmelCase   ,   return_labels=_lowerCAmelCase   )

			if return_labels:
				if model_class in get_values(_lowerCAmelCase   ):
					A					=   torch.zeros(
					    (self.model_tester.batch_size, self.model_tester.seq_length)   ,   dtype=torch.long   ,   device=_lowerCAmelCase   )
			return inputs_dict







		def 							A  (self    :   Dict   ):
			A					=   FunnelModelTester(self   )
			A					=   ConfigTester(self   ,   config_class=_lowerCAmelCase   )







		def 							A  (self    :   Dict   ):
			self.config_tester.run_common_tests()







		def 							A  (self    :   Any   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*_lowerCAmelCase   )







		def 							A  (self    :   List[str]   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_pretraining(*_lowerCAmelCase   )







		def 							A  (self    :   Dict   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_lm(*_lowerCAmelCase   )







		def 							A  (self    :   Any   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_token_classification(*_lowerCAmelCase   )







		def 							A  (self    :   str   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_question_answering(*_lowerCAmelCase   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   Optional[int]   ):
			if hasattr(_lowerCAmelCase   ,   """weight"""   ) and module.weight is not None:
				module.weight.data.fill_(3   )
			if hasattr(_lowerCAmelCase   ,   """bias"""   ) and module.bias is not None:
				module.bias.data.fill_(3   )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
				if hasattr(_lowerCAmelCase   ,   _lowerCAmelCase   ) and getattr(_lowerCAmelCase   ,   _lowerCAmelCase   ) is not None:
					A					=   getattr(_lowerCAmelCase   ,   _lowerCAmelCase   )
					weight.data.fill_(3   )






@require_torch
class 				__UpperCAmelCase	(							A__		,       unittest.TestCase    ):







		'''simple docstring'''







		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     (
		    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
		)







		def 							A  (self    :   List[str]   ):
			A					=   FunnelModelTester(self   ,   base=_lowerCAmelCase   )
			A					=   ConfigTester(self   ,   config_class=_lowerCAmelCase   )







		def 							A  (self    :   str   ):
			self.config_tester.run_common_tests()







		def 							A  (self    :   List[Any]   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_base_model(*_lowerCAmelCase   )







		def 							A  (self    :   Tuple   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_sequence_classification(*_lowerCAmelCase   )







		def 							A  (self    :   Dict   ):
			A					=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_multiple_choice(*_lowerCAmelCase   )







		def 							A  (self    :   List[str]   ):
			A	,     A					=   self.model_tester.prepare_config_and_inputs_for_common()
			A					=   True

			for model_class in self.all_model_classes:
				if model_class.__name__ == "FunnelBaseModel":
					continue
				A					=   model_class(_lowerCAmelCase   )
				model.to(_lowerCAmelCase   )
				model.train()
				A					=   self._prepare_for_class(_lowerCAmelCase   ,   _lowerCAmelCase   ,   return_labels=_lowerCAmelCase   )
				A					=   model(**_lowerCAmelCase   ).loss
				loss.backward()







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   Any   ):
			if hasattr(_lowerCAmelCase   ,   """weight"""   ) and module.weight is not None:
				module.weight.data.fill_(3   )
			if hasattr(_lowerCAmelCase   ,   """bias"""   ) and module.bias is not None:
				module.bias.data.fill_(3   )

			for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
				if hasattr(_lowerCAmelCase   ,   _lowerCAmelCase   ) and getattr(_lowerCAmelCase   ,   _lowerCAmelCase   ) is not None:
					A					=   getattr(_lowerCAmelCase   ,   _lowerCAmelCase   )
					weight.data.fill_(3   )






@require_torch
@require_sentencepiece
@require_tokenizers
class 				__UpperCAmelCase	(							unittest.TestCase    ):







		'''simple docstring'''







		def 							A  (self    :   Union[str, Any]   ):
			A					=   13
			A					=   7
			A					=   torch.arange(0   ,   batch_size * sequence_length   ).long().reshape(_lowerCAmelCase   ,   _lowerCAmelCase   )
			A					=   [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
			A					=   torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]   )

			A					=   FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""   )
			A					=   model(_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )[0].abs()

			A					=   torch.tensor(2_344.8_352   )
			A					=   torch.tensor(0.8_052   )
			self.assertTrue(torch.allclose(output.sum()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )
			self.assertTrue(torch.allclose(output.mean()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )

			A					=   torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]   )
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   token_type_ids=_lowerCAmelCase   )[0].abs()

			A					=   torch.tensor(2_343.8_425   )
			A					=   torch.tensor(0.8_049   )
			self.assertTrue(torch.allclose(output.sum()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )
			self.assertTrue(torch.allclose(output.mean()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )







		@slow
		def 							A  (self    :   Union[str, Any]   ):
			A					=   FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""   )
			A					=   FunnelModel.from_pretrained("""huggingface/funnel-small"""   )
			A					=   tokenizer("""Hello! I am the Funnel Transformer model."""   ,   return_tensors="""pt"""   )
			A					=   model(**_lowerCAmelCase   )[0]

			A					=   torch.tensor(235.7_246   )
			A					=   torch.tensor(0.0_256   )
			self.assertTrue(torch.allclose(output.sum()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )
			self.assertTrue(torch.allclose(output.mean()   ,   _lowerCAmelCase   ,   atol=1e-4   )   )








'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCamelCase					:						Dict					     =							logging.get_logger(__name__)

_lowerCamelCase					:						Dict					     =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase					:						str					     =							{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_lowerCamelCase					:						Union[str, Any]					     =							{
    'openai-gpt': 512,
}




def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   text.strip()
	if not text:
		return []
	A					=   text.split()
	return tokens



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		def __init__(self    :   Optional[int]   ,   _lowerCAmelCase    :   int=True   ,   _lowerCAmelCase    :   List[str]=None   ,   _lowerCAmelCase    :   str=True   ,   _lowerCAmelCase    :   int=None   ,   _lowerCAmelCase    :   Optional[Any]=True   ,   ):
			if never_split is None:
				A					=   []
			A					=   do_lower_case
			A					=   set(_lowerCAmelCase   )
			A					=   tokenize_chinese_chars
			A					=   strip_accents
			A					=   do_split_on_punc







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Any=None   ):
			A					=   self.never_split.union(set(_lowerCAmelCase   )   ) if never_split else self.never_split
			A					=   self._clean_text(_lowerCAmelCase   )

			# This was added on November 1st, 2018 for the multilingual and Chinese
			# models. This is also applied to the English models now, but it doesn't
			# matter since the English models were not trained on any Chinese data
			# and generally don't have any Chinese data in them (there are Chinese
			# characters in the vocabulary because Wikipedia does have some Chinese
			# words in the English Wikipedia.).
			if self.tokenize_chinese_chars:
				A					=   self._tokenize_chinese_chars(_lowerCAmelCase   )
			# prevents treating the same character with different unicode codepoints as different characters
			A					=   unicodedata.normalize("""NFC"""   ,   _lowerCAmelCase   )
			A					=   whitespace_tokenize(_lowerCAmelCase   )
			A					=   []
			for token in orig_tokens:
				if token not in never_split:
					if self.do_lower_case:
						A					=   token.lower()
						if self.strip_accents is not False:
							A					=   self._run_strip_accents(_lowerCAmelCase   )
					elif self.strip_accents:
						A					=   self._run_strip_accents(_lowerCAmelCase   )
				split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase   ,   _lowerCAmelCase   )   )

			A					=   whitespace_tokenize(""" """.join(_lowerCAmelCase   )   )
			return output_tokens







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   unicodedata.normalize("""NFD"""   ,   _lowerCAmelCase   )
			A					=   []
			for char in text:
				A					=   unicodedata.category(_lowerCAmelCase   )
				if cat == "Mn":
					continue
				output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   List[str]=None   ):
			if not self.do_split_on_punc or (never_split is not None and text in never_split):
				return [text]
			A					=   list(_lowerCAmelCase   )
			A					=   0
			A					=   True
			A					=   []
			while i < len(_lowerCAmelCase   ):
				A					=   chars[i]
				if _is_punctuation(_lowerCAmelCase   ):
					output.append([char]   )
					A					=   True
				else:
					if start_new_word:
						output.append([]   )
					A					=   False
					output[-1].append(_lowerCAmelCase   )
				i += 1

			return ["".join(_lowerCAmelCase   ) for x in output]







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if self._is_chinese_char(_lowerCAmelCase   ):
					output.append(""" """   )
					output.append(_lowerCAmelCase   )
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Any   ):
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0X4_e00 and cp <= 0X9_fff)
			    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
			    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
			    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
			    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
			    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
			    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
			    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
			):  #
				return True

			return False







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if cp == 0 or cp == 0Xf_ffd or _is_control(_lowerCAmelCase   ):
					continue
				if _is_whitespace(_lowerCAmelCase   ):
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )






def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   set()
	A					=   word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)     )
		A					=   char
	return pairs




def  __a			(  UpperCAmelCase     )   ->List[str]:



	"""simple docstring"""







	A					=   text.replace("""—"""							,      """-"""     )
	A					=   text.replace("""–"""							,      """-"""     )
	A					=   text.replace("""―"""							,      """-"""     )
	A					=   text.replace("""…"""							,      """..."""     )
	A					=   text.replace("""´"""							,      """'"""     )
	A					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""							,      R""" \1 """							,      UpperCAmelCase     )
	A					=   re.sub(R"""\s*\n\s*"""							,      """ \n """							,      UpperCAmelCase     )
	A					=   re.sub(R"""[^\S\n]+"""							,      """ """							,      UpperCAmelCase     )
	return text.strip()



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     VOCAB_FILES_NAMES
		__lowerCAmelCase								=     PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase								=     ['''input_ids''', '''attention_mask''']







		def __init__(self    :   Dict   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Dict="<unk>"   ,   **_lowerCAmelCase    :   Dict   ):
			super().__init__(unk_token=_lowerCAmelCase   ,   **_lowerCAmelCase   )

			try:
				import ftfy
				from spacy.lang.en import English

				A					=   English()
				A					=   _nlp.tokenizer
				A					=   ftfy.fix_text
			except ImportError:
				logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""   )
				A					=   BasicTokenizer(do_lower_case=_lowerCAmelCase   )
				A					=   None

			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as vocab_handle:
				A					=   json.load(_lowerCAmelCase   )
			A					=   {v: k for k, v in self.encoder.items()}
			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as merges_handle:
				A					=   merges_handle.read().split("""\n"""   )[1:-1]
			A					=   [tuple(merge.split()   ) for merge in merges]
			A					=   dict(zip(_lowerCAmelCase   ,   range(len(_lowerCAmelCase   )   )   )   )
			A					=   {}







		@property
		def 							A  (self    :   Dict   ):
			return True







		@property
		def 							A  (self    :   Dict   ):
			return len(self.encoder   )







		def 							A  (self    :   str   ):
			return dict(self.encoder   ,   **self.added_tokens_encoder   )







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   tuple(token[:-1]   ) + (token[-1] + """</w>""",)
			if token in self.cache:
				return self.cache[token]
			A					=   get_pairs(_lowerCAmelCase   )

			if not pairs:
				return token + "</w>"

			while True:
				A					=   min(_lowerCAmelCase   ,   key=lambda _lowerCAmelCase   : self.bpe_ranks.get(_lowerCAmelCase   ,   float("""inf"""   )   )   )
				if bigram not in self.bpe_ranks:
					break
				A	,     A					=   bigram
				A					=   []
				A					=   0
				while i < len(_lowerCAmelCase   ):
					try:
						A					=   word.index(_lowerCAmelCase   ,   _lowerCAmelCase   )
					except ValueError:
						new_word.extend(word[i:]   )
						break
					else:
						new_word.extend(word[i:j]   )
						A					=   j

					if word[i] == first and i < len(_lowerCAmelCase   ) - 1 and word[i + 1] == second:
						new_word.append(first + second   )
						i += 2
					else:
						new_word.append(word[i]   )
						i += 1
				A					=   tuple(_lowerCAmelCase   )
				A					=   new_word
				if len(_lowerCAmelCase   ) == 1:
					break
				else:
					A					=   get_pairs(_lowerCAmelCase   )
			A					=   """ """.join(_lowerCAmelCase   )
			if word == "\n  </w>":
				A					=   """\n</w>"""
			A					=   word
			return word







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   []
			if self.fix_text is None:
				# Using BERT's BasicTokenizer
				A					=   self.nlp.tokenize(_lowerCAmelCase   )
				for token in text:
					split_tokens.extend(list(self.bpe(_lowerCAmelCase   ).split(""" """   )   )   )
			else:
				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
				A					=   self.nlp(text_standardize(self.fix_text(_lowerCAmelCase   )   )   )
				for token in text:
					split_tokens.extend(list(self.bpe(token.text.lower()   ).split(""" """   )   )   )
			return split_tokens







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Dict   ):
			return self.encoder.get(_lowerCAmelCase   ,   self.encoder.get(self.unk_token   )   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Tuple   ):
			return self.decoder.get(_lowerCAmelCase   ,   self.unk_token   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   int   ):
			A					=   """""".join(_lowerCAmelCase   ).replace("""</w>"""   ,   """ """   ).strip()
			return out_string







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Optional[str] = None   ):
			if not os.path.isdir(_lowerCAmelCase   ):
				logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
				return
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
				f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=_lowerCAmelCase   ,   ensure_ascii=_lowerCAmelCase   ) + """\n"""   )

			A					=   0
			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as writer:
				writer.write("""#version: 0.2\n"""   )
				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda _lowerCAmelCase   : kv[1]   ):
					if index != token_index:
						logger.warning(
						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
						    """ Please check that the tokenizer is not corrupted!"""   )
						A					=   token_index
					writer.write(""" """.join(_lowerCAmelCase   ) + """\n"""   )
					index += 1

			return vocab_file, merge_file

