"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

_SCREAMING_SNAKE_CASE	:		int   =   logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE	:		str   =   {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_SCREAMING_SNAKE_CASE	:		Dict   =   {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

_SCREAMING_SNAKE_CASE	:		int   =   {
    '''gpt2''': 1_0_2_4,
    '''gpt2-medium''': 1_0_2_4,
    '''gpt2-large''': 1_0_2_4,
    '''gpt2-xl''': 1_0_2_4,
    '''distilgpt2''': 1_0_2_4,
}



@lru_cache()
def       A__    (	)  ->							Optional[int]:
   __lowerCamelCase    		=					(
       list(range(ord('!'						)      , ord('~'						) + 1						)						) + list(range(ord('¡'						)      , ord('¬'						) + 1						)						) + list(range(ord('®'						)      , ord('ÿ'						) + 1						)						)
   )
   __lowerCamelCase    		=					bs[:]
   __lowerCamelCase    		=					0
   for b in range(2**8						):
      if b not in bs:
         bs.append(lowercase_						)
         cs.append(2**8 + n						)
         n += 1
   __lowerCamelCase    		=					[chr(lowercase_						) for n in cs]
   return dict(zip(lowercase_      , lowercase_						)						)



def       A__    (	lowercase_     :	int						)  ->							Optional[Any]:
   __lowerCamelCase    		=					set()
   __lowerCamelCase    		=					word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)						)
      __lowerCamelCase    		=					char
   return pairs






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							Tuple    =    VOCAB_FILES_NAMES
   __UpperCAmelCase	:							int    =    PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase	:							Union[str, Any]    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase	:							Dict    =    ["input_ids", "attention_mask"]







   def __init__(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE="replace" ,				_SCREAMING_SNAKE_CASE="<|endoftext|>" ,				_SCREAMING_SNAKE_CASE="<|endoftext|>" ,				_SCREAMING_SNAKE_CASE="<|endoftext|>" ,				_SCREAMING_SNAKE_CASE=None ,				_SCREAMING_SNAKE_CASE=False ,				_SCREAMING_SNAKE_CASE=False ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      __lowerCamelCase    		=					AddedToken(_SCREAMING_SNAKE_CASE ,				lstrip=_SCREAMING_SNAKE_CASE ,				rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else bos_token
      __lowerCamelCase    		=					AddedToken(_SCREAMING_SNAKE_CASE ,				lstrip=_SCREAMING_SNAKE_CASE ,				rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else eos_token
      __lowerCamelCase    		=					AddedToken(_SCREAMING_SNAKE_CASE ,				lstrip=_SCREAMING_SNAKE_CASE ,				rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else unk_token
      __lowerCamelCase    		=					AddedToken(_SCREAMING_SNAKE_CASE ,				lstrip=_SCREAMING_SNAKE_CASE ,				rstrip=_SCREAMING_SNAKE_CASE	) if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else pad_token
      super().__init__(
          errors=_SCREAMING_SNAKE_CASE ,				unk_token=_SCREAMING_SNAKE_CASE ,				bos_token=_SCREAMING_SNAKE_CASE ,				eos_token=_SCREAMING_SNAKE_CASE ,				pad_token=_SCREAMING_SNAKE_CASE ,				add_prefix_space=_SCREAMING_SNAKE_CASE ,				add_bos_token=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE ,				)
      __lowerCamelCase    		=					add_bos_token

      with open(_SCREAMING_SNAKE_CASE ,				encoding='utf-8'	) as vocab_handle:
         __lowerCamelCase    		=					json.load(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					{v: k for k, v in self.encoder.items()}
      __lowerCamelCase    		=					errors  # how to handle errors in decoding
      __lowerCamelCase    		=					bytes_to_unicode()
      __lowerCamelCase    		=					{v: k for k, v in self.byte_encoder.items()}
      with open(_SCREAMING_SNAKE_CASE ,				encoding='utf-8'	) as merges_handle:
         __lowerCamelCase    		=					merges_handle.read().split('\n'	)[1:-1]
      __lowerCamelCase    		=					[tuple(merge.split()	) for merge in bpe_merges]
      __lowerCamelCase    		=					dict(zip(_SCREAMING_SNAKE_CASE ,				range(len(_SCREAMING_SNAKE_CASE	)	)	)	)
      __lowerCamelCase    		=					{}
      __lowerCamelCase    		=					add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      __lowerCamelCase    		=					re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)







   @property
   def    _A				(	self	):


      '''simple docstring'''

      return len(self.encoder	)







   def    _A				(	self	):


      '''simple docstring'''

      return dict(self.encoder ,				**self.added_tokens_encoder	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      if token in self.cache:
         return self.cache[token]
      __lowerCamelCase    		=					tuple(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					get_pairs(_SCREAMING_SNAKE_CASE	)

      if not pairs:
         return token

      while True:
         __lowerCamelCase    		=					min(_SCREAMING_SNAKE_CASE ,				key=lambda _SCREAMING_SNAKE_CASE	: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE ,				float('inf'	)	)	)
         if bigram not in self.bpe_ranks:
            break
         __lowerCamelCase		,__lowerCamelCase    		=					bigram
         __lowerCamelCase    		=					[]
         __lowerCamelCase    		=					0
         while i < len(_SCREAMING_SNAKE_CASE	):
            try:
               __lowerCamelCase    		=					word.index(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
            except ValueError:
               new_word.extend(word[i:]	)
               break
            else:
               new_word.extend(word[i:j]	)
               __lowerCamelCase    		=					j

            if word[i] == first and i < len(_SCREAMING_SNAKE_CASE	) - 1 and word[i + 1] == second:
               new_word.append(first + second	)
               i += 2
            else:
               new_word.append(word[i]	)
               i += 1
         __lowerCamelCase    		=					tuple(_SCREAMING_SNAKE_CASE	)
         __lowerCamelCase    		=					new_word
         if len(_SCREAMING_SNAKE_CASE	) == 1:
            break
         else:
            __lowerCamelCase    		=					get_pairs(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					' '.join(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					word
      return word







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE=None	):


      '''simple docstring'''

      if self.add_bos_token:
         __lowerCamelCase    		=					[self.bos_token_id]
      else:
         __lowerCamelCase    		=					[]

      __lowerCamelCase    		=					bos_token_ids + token_ids_a

      if token_ids_a is None:
         return output

      return output + bos_token_ids + token_ids_a







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = False	):


      '''simple docstring'''

      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=_SCREAMING_SNAKE_CASE ,				token_ids_a=_SCREAMING_SNAKE_CASE ,				already_has_special_tokens=_SCREAMING_SNAKE_CASE	)

      if not self.add_bos_token:
         return super().get_special_tokens_mask(
             token_ids_a=_SCREAMING_SNAKE_CASE ,				token_ids_a=_SCREAMING_SNAKE_CASE ,				already_has_special_tokens=_SCREAMING_SNAKE_CASE	)

      if token_ids_a is None:
         return [1] + ([0] * len(_SCREAMING_SNAKE_CASE	))
      return [1] + ([0] * len(_SCREAMING_SNAKE_CASE	)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE	))







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					[]
      for token in re.findall(self.pat ,				_SCREAMING_SNAKE_CASE	):
         __lowerCamelCase    		=					''.join(
             self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_SCREAMING_SNAKE_CASE	).split(' '	)	)
      return bpe_tokens







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      return self.encoder.get(_SCREAMING_SNAKE_CASE ,				self.encoder.get(self.unk_token	)	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      return self.decoder.get(_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					''.join(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8' ,				errors=self.errors	)
      return text







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None	):


      '''simple docstring'''

      if not os.path.isdir(_SCREAMING_SNAKE_CASE	):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
         return
      __lowerCamelCase    		=					os.path.join(
          _SCREAMING_SNAKE_CASE ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
      __lowerCamelCase    		=					os.path.join(
          _SCREAMING_SNAKE_CASE ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

      with open(_SCREAMING_SNAKE_CASE ,				'w' ,				encoding='utf-8'	) as f:
         f.write(json.dumps(self.encoder ,				indent=2 ,				sort_keys=_SCREAMING_SNAKE_CASE ,				ensure_ascii=_SCREAMING_SNAKE_CASE	) + '\n'	)

      __lowerCamelCase    		=					0
      with open(_SCREAMING_SNAKE_CASE ,				'w' ,				encoding='utf-8'	) as writer:
         writer.write('#version: 0.2\n'	)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,				key=lambda _SCREAMING_SNAKE_CASE	: kv[1]	):
            if index != token_index:
               logger.warning(
                   F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                   ' Please check that the tokenizer is not corrupted!'	)
               __lowerCamelCase    		=					token_index
            writer.write(' '.join(_SCREAMING_SNAKE_CASE	) + '\n'	)
            index += 1

      return vocab_file, merge_file







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE=False ,				**_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					kwargs.pop('add_prefix_space' ,				self.add_prefix_space	)
      if is_split_into_words or add_prefix_space:
         __lowerCamelCase    		=					' ' + text
      return (text, kwargs)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					[]
      for is_user, text in conversation.iter_texts():
         input_ids.extend(self.encode(_SCREAMING_SNAKE_CASE ,				add_special_tokens=_SCREAMING_SNAKE_CASE	) + [self.eos_token_id]	)
      if len(_SCREAMING_SNAKE_CASE	) > self.model_max_length:
         __lowerCamelCase    		=					input_ids[-self.model_max_length :]
      return input_ids









import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_PRETRAINING_MAPPING,
         FunnelBaseModel,
         FunnelForMaskedLM,
         FunnelForMultipleChoice,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForSequenceClassification,
         FunnelForTokenClassification,
         FunnelModel,
     )





class    __UpperCAmelCase	:



   def __init__(				self				:	List[Any],						__A				:	str,						__A				:	Union[str, Any]=1_3,						__A				:	Union[str, Any]=7,						__A				:	List[str]=True,						__A				:	Optional[Any]=True,						__A				:	Tuple=True,						__A				:	Any=True,						__A				:	Dict=9_9,						__A				:	Union[str, Any]=[1, 1, 2],						__A				:	Tuple=1,						__A				:	int=3_2,						__A				:	List[str]=4,						__A				:	List[str]=8,						__A				:	int=3_7,						__A				:	Optional[int]="gelu_new",						__A				:	List[Any]=0.1,						__A				:	str=0.1,						__A				:	int=0.0,						__A				:	Union[str, Any]=5_1_2,						__A				:	int=3,						__A				:	str=0.0_2,						__A				:	Tuple=3,						__A				:	Any=4,						__A				:	Any=None,						__A				:	Any=False,						):
     UpperCAmelCase			:				str										=					parent
     UpperCAmelCase			:				Optional[Any]										=					batch_size
     UpperCAmelCase			:				str										=					seq_length
     UpperCAmelCase			:				str										=					is_training
     UpperCAmelCase			:				Tuple										=					use_input_mask
     UpperCAmelCase			:				Union[str, Any]										=					use_token_type_ids
     UpperCAmelCase			:				Dict										=					use_labels
     UpperCAmelCase			:				Union[str, Any]										=					vocab_size
     UpperCAmelCase			:				Tuple										=					block_sizes
     UpperCAmelCase			:				List[Any]										=					num_decoder_layers
     UpperCAmelCase			:				Union[str, Any]										=					d_model
     UpperCAmelCase			:				Tuple										=					n_head
     UpperCAmelCase			:				Dict										=					d_head
     UpperCAmelCase			:				Optional[int]										=					d_inner
     UpperCAmelCase			:				Optional[int]										=					hidden_act
     UpperCAmelCase			:				int										=					hidden_dropout
     UpperCAmelCase			:				Any										=					attention_dropout
     UpperCAmelCase			:				str										=					activation_dropout
     UpperCAmelCase			:				Dict										=					max_position_embeddings
     UpperCAmelCase			:				Optional[Any]										=					type_vocab_size
     UpperCAmelCase			:				Any										=					2
     UpperCAmelCase			:				List[Any]										=					num_labels
     UpperCAmelCase			:				str										=					num_choices
     UpperCAmelCase			:				str										=					scope
     UpperCAmelCase			:				int										=					initializer_std

     # Used in the tests to check the size of the first attention layer
     UpperCAmelCase			:				int										=					n_head
     # Used in the tests to check the size of the first hidden state
     UpperCAmelCase			:				Dict										=					self.d_model
     # Used in the tests to check the number of output hidden states/attentions
     UpperCAmelCase			:				List[str]										=					sum(self.block_sizes		) + (0 if base else self.num_decoder_layers)
     # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
     # the last hidden state of the first block (which is the first hidden state of the decoder).
     if not base:
       UpperCAmelCase			:				List[str]										=					self.num_hidden_layers + 2



   def 			__magic_name__       (				self				:	List[str]		):
     UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.vocab_size		)

     UpperCAmelCase			:				Optional[int]										=					None
     if self.use_input_mask:
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size, self.seq_length],						vocab_size=2		)

     UpperCAmelCase			:				int										=					None
     if self.use_token_type_ids:
       UpperCAmelCase			:				Dict										=					ids_tensor([self.batch_size, self.seq_length],						self.type_vocab_size		)

     UpperCAmelCase			:				List[Any]										=					None
     UpperCAmelCase			:				Optional[int]										=					None
     UpperCAmelCase			:				str										=					None
     if self.use_labels:
       UpperCAmelCase			:				List[Any]										=					ids_tensor([self.batch_size],						self.type_sequence_label_size		)
       UpperCAmelCase			:				Optional[int]										=					ids_tensor([self.batch_size, self.seq_length],						self.num_labels		)
       UpperCAmelCase			:				List[str]										=					ids_tensor([self.batch_size],						self.num_choices		)
       UpperCAmelCase			:				Tuple										=					ids_tensor([self.batch_size, self.seq_length],						1		)

     UpperCAmelCase			:				Dict										=					self.get_config()

     return (
         config,
         input_ids,
         token_type_ids,
         input_mask,
         sequence_labels,
         token_labels,
         choice_labels,
         fake_token_labels,
     )



   def 			__magic_name__       (				self				:	List[Any]		):
     return FunnelConfig(
         vocab_size=self.vocab_size,						block_sizes=self.block_sizes,						num_decoder_layers=self.num_decoder_layers,						d_model=self.d_model,						n_head=self.n_head,						d_head=self.d_head,						d_inner=self.d_inner,						hidden_act=self.hidden_act,						hidden_dropout=self.hidden_dropout,						attention_dropout=self.attention_dropout,						activation_dropout=self.activation_dropout,						max_position_embeddings=self.max_position_embeddings,						type_vocab_size=self.type_vocab_size,						initializer_std=self.initializer_std,						)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	List[str],						__A				:	Dict,						__A				:	Optional[int],						__A				:	List[Any],						__A				:	List[Any],						__A				:	Optional[int],						__A				:	Dict,						__A				:	Dict,						):
     UpperCAmelCase			:				List[str]										=					FunnelModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[int]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				Optional[int]										=					False
     UpperCAmelCase			:				List[str]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)

     UpperCAmelCase			:				List[str]										=					False
     UpperCAmelCase			:				Dict										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, self.seq_length, self.d_model)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	str,						__A				:	List[Any],						__A				:	Union[str, Any],						__A				:	Union[str, Any],						__A				:	Tuple,						__A				:	Tuple,						__A				:	List[Any],						__A				:	Dict,						):
     UpperCAmelCase			:				List[Any]										=					FunnelBaseModel(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				int										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)

     UpperCAmelCase			:				str										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 3, self.d_model)		)

     UpperCAmelCase			:				Tuple										=					False
     UpperCAmelCase			:				Optional[Any]										=					model(__A		)
     self.parent.assertEqual(result.last_hidden_state.shape,						(self.batch_size, 2, self.d_model)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Tuple,						__A				:	Dict,						__A				:	str,						__A				:	Dict,						__A				:	Union[str, Any],						__A				:	List[Any],						__A				:	Any,						__A				:	str,						):
     UpperCAmelCase			:				int										=					self.num_labels
     UpperCAmelCase			:				int										=					FunnelForPreTraining(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length)		)



   def 			__magic_name__       (				self				:	List[str],						__A				:	Optional[int],						__A				:	Union[str, Any],						__A				:	int,						__A				:	Union[str, Any],						__A				:	Optional[int],						__A				:	str,						__A				:	int,						__A				:	str,						):
     UpperCAmelCase			:				List[Any]										=					FunnelForMaskedLM(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[int]										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.vocab_size)		)



   def 			__magic_name__       (				self				:	Optional[int],						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Dict,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	Union[str, Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForSequenceClassification(__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Dict										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_labels)		)



   def 			__magic_name__       (				self				:	Union[str, Any],						__A				:	Any,						__A				:	List[str],						__A				:	str,						__A				:	List[Any],						__A				:	List[str],						__A				:	Optional[Any],						__A				:	List[Any],						__A				:	List[str],						):
     UpperCAmelCase			:				str										=					self.num_choices
     UpperCAmelCase			:				Optional[int]										=					FunnelForMultipleChoice(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				List[Any]										=					input_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					token_type_ids.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[Any]										=					input_mask.unsqueeze(1		).expand(-1,						self.num_choices,						-1		).contiguous()
     UpperCAmelCase			:				List[str]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						labels=__A,						)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.num_choices)		)



   def 			__magic_name__       (				self				:	Dict,						__A				:	Tuple,						__A				:	int,						__A				:	Any,						__A				:	Optional[Any],						__A				:	Optional[Any],						__A				:	Tuple,						__A				:	str,						__A				:	List[str],						):
     UpperCAmelCase			:				Union[str, Any]										=					self.num_labels
     UpperCAmelCase			:				Optional[int]										=					FunnelForTokenClassification(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Any										=					model(__A,						attention_mask=__A,						token_type_ids=__A,						labels=__A		)
     self.parent.assertEqual(result.logits.shape,						(self.batch_size, self.seq_length, self.num_labels)		)



   def 			__magic_name__       (				self				:	int,						__A				:	Optional[int],						__A				:	List[str],						__A				:	List[Any],						__A				:	int,						__A				:	str,						__A				:	Any,						__A				:	Union[str, Any],						__A				:	str,						):
     UpperCAmelCase			:				str										=					FunnelForQuestionAnswering(config=__A		)
     model.to(__A		)
     model.eval()
     UpperCAmelCase			:				Optional[Any]										=					model(
         __A,						attention_mask=__A,						token_type_ids=__A,						start_positions=__A,						end_positions=__A,						)
     self.parent.assertEqual(result.start_logits.shape,						(self.batch_size, self.seq_length)		)
     self.parent.assertEqual(result.end_logits.shape,						(self.batch_size, self.seq_length)		)





   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Dict										=					self.prepare_config_and_inputs()
     (
         (
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		(
         UpperCAmelCase
     )					,		
     )			:				Optional[int]										=					config_and_inputs
     UpperCAmelCase			:				int										=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
     return config, inputs_dict






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   UpperCamelCase  	=	(
       {
           """feature-extraction""": (FunnelBaseModel, FunnelModel),
           """fill-mask""": FunnelForMaskedLM,
           """question-answering""": FunnelForQuestionAnswering,
           """text-classification""": FunnelForSequenceClassification,
           """token-classification""": FunnelForTokenClassification,
           """zero-shot""": FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )



   def 			__magic_name__       (				self				:	Dict,						__A				:	List[Any],						__A				:	str,						__A				:	Dict=False		):
     UpperCAmelCase			:				int										=					super()._prepare_for_class(__A,						__A,						return_labels=__A		)

     if return_labels:
       if model_class in get_values(__A		):
         UpperCAmelCase			:				Tuple										=					torch.zeros(
             (self.model_tester.batch_size, self.model_tester.seq_length),						dtype=torch.long,						device=__A		)
     return inputs_dict



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self		)
     UpperCAmelCase			:				str										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Dict		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				List[Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_pretraining(*__A		)



   def 			__magic_name__       (				self				:	Dict		):
     UpperCAmelCase			:				str										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*__A		)



   def 			__magic_name__       (				self				:	str		):
     UpperCAmelCase			:				Any										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*__A		)



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				int										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*__A		)





   def 			__magic_name__       (				self				:	Any,						__A				:	List[Any]		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
class    __UpperCAmelCase	(			lowerCamelCase__     ,    unittest.TestCase		):
   UpperCamelCase  	=	False
   UpperCamelCase  	=	False
   UpperCamelCase  	=	(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )



   def 			__magic_name__       (				self				:	List[Any]		):
     UpperCAmelCase			:				Optional[int]										=					FunnelModelTester(self,						base=__A		)
     UpperCAmelCase			:				Tuple										=					ConfigTester(self,						config_class=__A		)



   def 			__magic_name__       (				self				:	Union[str, Any]		):
     self.config_tester.run_common_tests()



   def 			__magic_name__       (				self				:	Any		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_base_model(*__A		)



   def 			__magic_name__       (				self				:	Optional[Any]		):
     UpperCAmelCase			:				Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase			:				Tuple										=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*__A		)



   def 			__magic_name__       (				self				:	Optional[int]		):
     UpperCAmelCase					,		UpperCAmelCase			:				List[str]										=					self.model_tester.prepare_config_and_inputs_for_common()
     UpperCAmelCase			:				List[str]										=					True

     for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
         continue
       UpperCAmelCase			:				Any										=					model_class(__A		)
       model.to(__A		)
       model.train()
       UpperCAmelCase			:				str										=					self._prepare_for_class(__A,						__A,						return_labels=__A		)
       UpperCAmelCase			:				str										=					model(**__A		).loss
       loss.backward()





   def 			__magic_name__       (				self				:	str,						__A				:	Any		):
     if hasattr(__A,						'''weight'''		) and module.weight is not None:
       module.weight.data.fill_(3		)
     if hasattr(__A,						'''bias'''		) and module.bias is not None:
       module.bias.data.fill_(3		)

     for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__A,						__A		) and getattr(__A,						__A		) is not None:
         UpperCAmelCase			:				Optional[int]										=					getattr(__A,						__A		)
         weight.data.fill_(3		)






@require_torch
@require_sentencepiece
@require_tokenizers
class    __UpperCAmelCase	(			unittest.TestCase		):



   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				Optional[Any]										=					1_3
     UpperCAmelCase			:				Tuple										=					7
     UpperCAmelCase			:				Optional[Any]										=					torch.arange(0,						batch_size * sequence_length		).long().reshape(__A,						__A		)
     UpperCAmelCase			:				List[Any]										=					[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]		)

     UpperCAmelCase			:				Dict										=					FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''		)
     UpperCAmelCase			:				List[Any]										=					model(__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor(2_3_4_4.8_3_5_2		)
     UpperCAmelCase			:				List[Any]										=					torch.tensor(0.8_0_5_2		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)

     UpperCAmelCase			:				Union[str, Any]										=					torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]		)
     UpperCAmelCase			:				Union[str, Any]										=					model(__A,						attention_mask=__A,						token_type_ids=__A		)[0].abs()

     UpperCAmelCase			:				str										=					torch.tensor(2_3_4_3.8_4_2_5		)
     UpperCAmelCase			:				Optional[Any]										=					torch.tensor(0.8_0_4_9		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





   @slow
   def 			__magic_name__       (				self				:	int		):
     UpperCAmelCase			:				List[Any]										=					FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					FunnelModel.from_pretrained('''huggingface/funnel-small'''		)
     UpperCAmelCase			:				List[str]										=					tokenizer('''Hello! I am the Funnel Transformer model.''',						return_tensors='''pt'''		)
     UpperCAmelCase			:				Optional[Any]										=					model(**__A		)[0]

     UpperCAmelCase			:				int										=					torch.tensor(2_3_5.7_2_4_6		)
     UpperCAmelCase			:				Optional[int]										=					torch.tensor(0.0_2_5_6		)
     self.assertTrue(torch.allclose(output.sum(),						__A,						atol=1E-4		)		)
     self.assertTrue(torch.allclose(output.mean(),						__A,						atol=1E-4		)		)





