"""simple docstring"""




from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFFunnelBaseModel,
         TFFunnelForMaskedLM,
         TFFunnelForMultipleChoice,
         TFFunnelForPreTraining,
         TFFunnelForQuestionAnswering,
         TFFunnelForSequenceClassification,
         TFFunnelForTokenClassification,
         TFFunnelModel,
     )
class 						__A							:






     def __init__(		self				,     a__				,     a__=13				,     a__=7				,     a__=True				,     a__=True				,     a__=True				,     a__=True				,     a__=99				,     a__=[1, 1, 2]				,     a__=1				,     a__=32				,     a__=4				,     a__=8				,     a__=37				,     a__="gelu_new"				,     a__=0.1				,     a__=0.1				,     a__=0.0				,     a__=512				,     a__=3				,     a__=0.0_2				,     a__=3				,     a__=4				,     a__=None				,     a__=False				,     ):
       _lowerCAmelCase						:       Union[str, Any]  			=     parent
       _lowerCAmelCase						:       Optional[Any]  			=     batch_size
       _lowerCAmelCase						:       Optional[Any]  			=     seq_length
       _lowerCAmelCase						:       Union[str, Any]  			=     is_training
       _lowerCAmelCase						:       List[Any]  			=     use_input_mask
       _lowerCAmelCase						:       List[Any]  			=     use_token_type_ids
       _lowerCAmelCase						:       List[Any]  			=     use_labels
       _lowerCAmelCase						:       List[Any]  			=     vocab_size
       _lowerCAmelCase						:       Any  			=     block_sizes
       _lowerCAmelCase						:       List[str]  			=     num_decoder_layers
       _lowerCAmelCase						:       List[Any]  			=     d_model
       _lowerCAmelCase						:       List[Any]  			=     n_head
       _lowerCAmelCase						:       Any  			=     d_head
       _lowerCAmelCase						:       Union[str, Any]  			=     d_inner
       _lowerCAmelCase						:       Optional[int]  			=     hidden_act
       _lowerCAmelCase						:       Optional[Any]  			=     hidden_dropout
       _lowerCAmelCase						:       Optional[int]  			=     attention_dropout
       _lowerCAmelCase						:       int  			=     activation_dropout
       _lowerCAmelCase						:       str  			=     max_position_embeddings
       _lowerCAmelCase						:       Optional[Any]  			=     type_vocab_size
       _lowerCAmelCase						:       Optional[Any]  			=     2
       _lowerCAmelCase						:       List[str]  			=     num_labels
       _lowerCAmelCase						:       Dict  			=     num_choices
       _lowerCAmelCase						:       Optional[int]  			=     scope
       _lowerCAmelCase						:       List[str]  			=     initializer_std

       # Used in the tests to check the size of the first attention layer
       _lowerCAmelCase						:       Dict  			=     n_head
       # Used in the tests to check the size of the first hidden state
       _lowerCAmelCase						:       Union[str, Any]  			=     self.d_model
       # Used in the tests to check the number of output hidden states/attentions
       _lowerCAmelCase						:       List[Any]  			=     sum(self.block_sizes  ) + (0 if base else self.num_decoder_layers)
       # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
       # the last hidden state of the first block (which is the first hidden state of the decoder).
       if not base:
         _lowerCAmelCase						:       List[Any]  			=     self.num_hidden_layers + 2






     def     __A  (		self  ):
       _lowerCAmelCase						:       Union[str, Any]  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size  )

       _lowerCAmelCase						:       Dict  			=     None
       if self.use_input_mask:
         _lowerCAmelCase						:       Dict  			=     random_attention_mask([self.batch_size, self.seq_length]  )

       _lowerCAmelCase						:       Dict  			=     None
       if self.use_token_type_ids:
         _lowerCAmelCase						:       Any  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.type_vocab_size  )

       _lowerCAmelCase						:       Dict  			=     None
       _lowerCAmelCase						:       Dict  			=     None
       _lowerCAmelCase						:       Tuple  			=     None
       if self.use_labels:
         _lowerCAmelCase						:       Any  			=     ids_tensor([self.batch_size]				,     self.type_sequence_label_size  )
         _lowerCAmelCase						:       Tuple  			=     ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels  )
         _lowerCAmelCase						:       List[Any]  			=     ids_tensor([self.batch_size]				,     self.num_choices  )

       _lowerCAmelCase						:       Tuple  			=     FunnelConfig(
           vocab_size=self.vocab_size				,     block_sizes=self.block_sizes				,     num_decoder_layers=self.num_decoder_layers				,     d_model=self.d_model				,     n_head=self.n_head				,     d_head=self.d_head				,     d_inner=self.d_inner				,     hidden_act=self.hidden_act				,     hidden_dropout=self.hidden_dropout				,     attention_dropout=self.attention_dropout				,     activation_dropout=self.activation_dropout				,     max_position_embeddings=self.max_position_embeddings				,     type_vocab_size=self.type_vocab_size				,     initializer_std=self.initializer_std				,     )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
       )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       List[str]  			=     TFFunnelModel(config=a__  )
       _lowerCAmelCase						:       Any  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       List[Any]  			=     model(a__  )

       _lowerCAmelCase						:       Tuple  			=     [input_ids, input_mask]
       _lowerCAmelCase						:       List[str]  			=     model(a__  )

       _lowerCAmelCase						:       Any  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.d_model)  )

       _lowerCAmelCase						:       str  			=     False
       _lowerCAmelCase						:       List[str]  			=     TFFunnelModel(config=a__  )
       _lowerCAmelCase						:       Optional[int]  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.d_model)  )

       _lowerCAmelCase						:       int  			=     False
       _lowerCAmelCase						:       Tuple  			=     TFFunnelModel(config=a__  )
       _lowerCAmelCase						:       int  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.d_model)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Optional[Any]  			=     TFFunnelBaseModel(config=a__  )
       _lowerCAmelCase						:       Dict  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       Any  			=     model(a__  )

       _lowerCAmelCase						:       Optional[int]  			=     [input_ids, input_mask]
       _lowerCAmelCase						:       Any  			=     model(a__  )

       _lowerCAmelCase						:       Optional[int]  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, 2, self.d_model)  )

       _lowerCAmelCase						:       List[Any]  			=     False
       _lowerCAmelCase						:       int  			=     TFFunnelBaseModel(config=a__  )
       _lowerCAmelCase						:       List[str]  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, 3, self.d_model)  )

       _lowerCAmelCase						:       Any  			=     False
       _lowerCAmelCase						:       Optional[Any]  			=     TFFunnelBaseModel(config=a__  )
       _lowerCAmelCase						:       Dict  			=     model(a__  )
       self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, 2, self.d_model)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Union[str, Any]  			=     TFFunnelForPreTraining(config=a__  )
       _lowerCAmelCase						:       Union[str, Any]  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       Optional[int]  			=     model(a__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Any  			=     TFFunnelForMaskedLM(config=a__  )
       _lowerCAmelCase						:       List[str]  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       List[str]  			=     model(a__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.vocab_size)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Optional[int]  			=     self.num_labels
       _lowerCAmelCase						:       List[str]  			=     TFFunnelForSequenceClassification(config=a__  )
       _lowerCAmelCase						:       List[str]  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       List[Any]  			=     model(a__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_labels)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       List[str]  			=     self.num_choices
       _lowerCAmelCase						:       Dict  			=     TFFunnelForMultipleChoice(config=a__  )
       _lowerCAmelCase						:       Any  			=     tf.tile(tf.expand_dims(a__				,     1  )				,     (1, self.num_choices, 1)  )
       _lowerCAmelCase						:       int  			=     tf.tile(tf.expand_dims(a__				,     1  )				,     (1, self.num_choices, 1)  )
       _lowerCAmelCase						:       str  			=     tf.tile(tf.expand_dims(a__				,     1  )				,     (1, self.num_choices, 1)  )
       _lowerCAmelCase						:       List[str]  			=     {
           """input_ids""": multiple_choice_inputs_ids,
           """attention_mask""": multiple_choice_input_mask,
           """token_type_ids""": multiple_choice_token_type_ids,
       }
       _lowerCAmelCase						:       List[str]  			=     model(a__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_choices)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Tuple  			=     self.num_labels
       _lowerCAmelCase						:       List[Any]  			=     TFFunnelForTokenClassification(config=a__  )
       _lowerCAmelCase						:       Optional[Any]  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       Dict  			=     model(a__  )
       self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.num_labels)  )






     def     __A  (		self				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     a__				,     ):
       _lowerCAmelCase						:       Tuple  			=     TFFunnelForQuestionAnswering(config=a__  )
       _lowerCAmelCase						:       List[str]  			=     {"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}
       _lowerCAmelCase						:       Optional[Any]  			=     model(a__  )
       self.parent.assertEqual(result.start_logits.shape				,     (self.batch_size, self.seq_length)  )
       self.parent.assertEqual(result.end_logits.shape				,     (self.batch_size, self.seq_length)  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self.prepare_config_and_inputs()
       (
           (
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	(
           _lowerCAmelCase
       )			,	
       )						:       Union[str, Any]  			=     config_and_inputs
       _lowerCAmelCase						:       str  			=     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
       return config, inputs_dict







@require_tf
class 						__A							( SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_    ,	unittest.TestCase		):
     _UpperCamelCase							:			Dict							=		(
         (
             TFFunnelModel,
             TFFunnelForMaskedLM,
             TFFunnelForPreTraining,
             TFFunnelForQuestionAnswering,
             TFFunnelForTokenClassification,
         )
         if is_tf_available()
         else ()
     )
     _UpperCamelCase							:			int							=		(
         {
             "feature-extraction": (TFFunnelBaseModel, TFFunnelModel),
             "fill-mask": TFFunnelForMaskedLM,
             "question-answering": TFFunnelForQuestionAnswering,
             "text-classification": TFFunnelForSequenceClassification,
             "token-classification": TFFunnelForTokenClassification,
             "zero-shot": TFFunnelForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     _UpperCamelCase							:			List[str]							=		False
     _UpperCamelCase							:			List[str]							=		False






     def     __A  (		self  ):
       _lowerCAmelCase						:       Dict  			=     TFFunnelModelTester(self  )
       _lowerCAmelCase						:       Union[str, Any]  			=     ConfigTester(self				,     config_class=a__  )






     def     __A  (		self  ):
       self.config_tester.run_common_tests()






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Any  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[str]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*a__  )







@require_tf
class 						__A							( SCREAMING_SNAKE_CASE_    ,	unittest.TestCase		):
     _UpperCamelCase							:			List[str]							=		(
         (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
     )
     _UpperCamelCase							:			Optional[Any]							=		False
     _UpperCamelCase							:			Optional[Any]							=		False






     def     __A  (		self  ):
       _lowerCAmelCase						:       Tuple  			=     TFFunnelModelTester(self				,     base=a__  )
       _lowerCAmelCase						:       List[Any]  			=     ConfigTester(self				,     config_class=a__  )






     def     __A  (		self  ):
       self.config_tester.run_common_tests()






     def     __A  (		self  ):
       _lowerCAmelCase						:       Optional[int]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_base_model(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Union[str, Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*a__  )





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_a      :		List[Any]           =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a      :		Union[str, Any]           =   {
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

_a      :		Optional[Any]           =   {
    'google/electra-small-generator': 512,
    'google/electra-base-generator': 512,
    'google/electra-large-generator': 512,
    'google/electra-small-discriminator': 512,
    'google/electra-base-discriminator': 512,
    'google/electra-large-discriminator': 512,
}

_a      :		Any           =   {
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}
class 						__A							( SCREAMING_SNAKE_CASE_		):
     _UpperCamelCase							:			Tuple							=		VOCAB_FILES_NAMES
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_INIT_CONFIGURATION
     _UpperCamelCase							:			Union[str, Any]							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:			Optional[Any]							=		ElectraTokenizer






     def __init__(		self				,     a__=None				,     a__=None				,     a__=True				,     a__="[UNK]"				,     a__="[SEP]"				,     a__="[PAD]"				,     a__="[CLS]"				,     a__="[MASK]"				,     a__=True				,     a__=None				,     **a__				,     ):
       super().__init__(
           a__				,     tokenizer_file=a__				,     do_lower_case=a__				,     unk_token=a__				,     sep_token=a__				,     pad_token=a__				,     cls_token=a__				,     mask_token=a__				,     tokenize_chinese_chars=a__				,     strip_accents=a__				,     **a__				,     )

       _lowerCAmelCase						:       int  			=     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get("""lowercase"""				,     a__  ) != do_lower_case
           or normalizer_state.get("""strip_accents"""				,     a__  ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""				,     a__  ) != tokenize_chinese_chars
       ):
         _lowerCAmelCase						:       Dict  			=     getattr(a__				,     normalizer_state.pop("""type"""  )  )
         _lowerCAmelCase						:       int  			=     do_lower_case
         _lowerCAmelCase						:       str  			=     strip_accents
         _lowerCAmelCase						:       Dict  			=     tokenize_chinese_chars
         _lowerCAmelCase						:       str  			=     normalizer_class(**a__  )

       _lowerCAmelCase						:       List[str]  			=     do_lower_case






     def     __A  (		self				,     a__				,     a__=None  ):
       _lowerCAmelCase						:       int  			=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       List[str]  			=     [self.sep_token_id]
       _lowerCAmelCase						:       Any  			=     [self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self._tokenizer.model.save(a__				,     name=a__  )
       return tuple(a__  )




