

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


lowerCamelCase 	=   logging.get_logger(__name__)

lowerCamelCase 	=   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase 	=   {
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''allenai/longformer-base-4096''': (
            '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase 	=   {
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}







class 		__magic_name__     (				lowerCamelCase__				):

   '''simple docstring'''




   lowerCamelCase__					:      Optional[int]        =  VOCAB_FILES_NAMES
   lowerCamelCase__					:      List[str]        =  PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__					:      Tuple        =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__					:      Dict        =  ['input_ids', 'attention_mask']
   lowerCamelCase__					:      Any        =  LongformerTokenizer



   def __init__(							self,       lowercase_=None,       lowercase_=None,       lowercase_=None,       lowercase_="replace",       lowercase_="<s>",       lowercase_="</s>",       lowercase_="</s>",       lowercase_="<s>",       lowercase_="<unk>",       lowercase_="<pad>",       lowercase_="<mask>",       lowercase_=False,       lowercase_=True,       **lowercase_,       )				->       List[str]:






     """simple docstring"""






     super().__init__(
         lowercase_,       lowercase_,       tokenizer_file=lowercase_,       errors=lowercase_,       bos_token=lowercase_,       eos_token=lowercase_,       sep_token=lowercase_,       cls_token=lowercase_,       unk_token=lowercase_,       pad_token=lowercase_,       mask_token=lowercase_,       add_prefix_space=lowercase_,       trim_offsets=lowercase_,       **lowercase_,       )

     a__    =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()			)
     if pre_tok_state.get('''add_prefix_space''',       lowercase_			) != add_prefix_space:
       a__    =getattr(lowercase_,       pre_tok_state.pop('''type'''			)			)
       a__    =add_prefix_space
       a__    =pre_tok_class(**lowercase_			)

     a__    =add_prefix_space

     a__    ='''post_processor'''
     a__    =getattr(self.backend_tokenizer,       lowercase_,       lowercase_			)
     if tokenizer_component_instance:
       a__    =json.loads(tokenizer_component_instance.__getstate__()			)

       # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
       if "sep" in state:
         a__    =tuple(state['''sep''']			)
       if "cls" in state:
         a__    =tuple(state['''cls''']			)

       a__    =False

       if state.get('''add_prefix_space''',       lowercase_			) != add_prefix_space:
         a__    =add_prefix_space
         a__    =True

       if state.get('''trim_offsets''',       lowercase_			) != trim_offsets:
         a__    =trim_offsets
         a__    =True

       if changes_to_apply:
         a__    =getattr(lowercase_,       state.pop('''type'''			)			)
         a__    =component_class(**lowercase_			)
         setattr(self.backend_tokenizer,       lowercase_,       lowercase_			)



   @property
   def 					_UpperCAmelCase					(							self			)				->       str:






     """simple docstring"""






     if self._mask_token is None:
       if self.verbose:
         logger.error('''Using mask_token, but it is not set yet.'''			)
       return None
     return str(self._mask_token			)



   @mask_token.setter
   def 					_UpperCAmelCase					(							self,       lowercase_			)				->       List[Any]:






     """simple docstring"""






     a__    =AddedToken(lowercase_,       lstrip=lowercase_,       rstrip=lowercase_			) if isinstance(lowercase_,       lowercase_			) else value
     a__    =value



   def 					_UpperCAmelCase					(							self,       *lowercase_,       **lowercase_			)				->       BatchEncoding:






     """simple docstring"""






     a__    =kwargs.get('''is_split_into_words''',       lowercase_			)
     assert self.add_prefix_space or not is_split_into_words, (
         F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
         "to use it with pretokenized inputs."
     )

     return super()._batch_encode_plus(*lowercase_,       **lowercase_			)



   def 					_UpperCAmelCase					(							self,       *lowercase_,       **lowercase_			)				->       BatchEncoding:






     """simple docstring"""






     a__    =kwargs.get('''is_split_into_words''',       lowercase_			)

     assert self.add_prefix_space or not is_split_into_words, (
         F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
         "to use it with pretokenized inputs."
     )

     return super()._encode_plus(*lowercase_,       **lowercase_			)



   def 					_UpperCAmelCase					(							self,       lowercase_,       lowercase_ = None			)				->       Tuple[str]:






     """simple docstring"""






     a__    =self._tokenizer.model.save(lowercase_,       name=lowercase_			)
     return tuple(lowercase_			)



   def 					_UpperCAmelCase					(							self,       lowercase_,       lowercase_=None			)				->       List[Any]:






     """simple docstring"""






     a__    =[self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
       return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



   def 					_UpperCAmelCase					(							self,       lowercase_,       lowercase_ = None			)				->       List[int]:






     """simple docstring"""






     a__    =[self.sep_token_id]
     a__    =[self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]



import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
       from transformers.models.electra.modeling_flax_electra import (
           FlaxElectraForCausalLM,
           FlaxElectraForMaskedLM,
           FlaxElectraForMultipleChoice,
           FlaxElectraForPreTraining,
           FlaxElectraForQuestionAnswering,
           FlaxElectraForSequenceClassification,
           FlaxElectraForTokenClassification,
           FlaxElectraModel,
       )







class 		__magic_name__     (				unittest.TestCase				):

   '''simple docstring'''




   def __init__(							self,       lowercase_,       lowercase_=13,       lowercase_=7,       lowercase_=True,       lowercase_=True,       lowercase_=True,       lowercase_=True,       lowercase_=99,       lowercase_=24,       lowercase_=32,       lowercase_=5,       lowercase_=4,       lowercase_=37,       lowercase_="gelu",       lowercase_=0.1,       lowercase_=0.1,       lowercase_=512,       lowercase_=16,       lowercase_=2,       lowercase_=0.02,       lowercase_=4,       )				->       int:






     """simple docstring"""






     a__    =parent
     a__    =batch_size
     a__    =seq_length
     a__    =is_training
     a__    =use_attention_mask
     a__    =use_token_type_ids
     a__    =use_labels
     a__    =vocab_size
     a__    =hidden_size
     a__    =embedding_size
     a__    =num_hidden_layers
     a__    =num_attention_heads
     a__    =intermediate_size
     a__    =hidden_act
     a__    =hidden_dropout_prob
     a__    =attention_probs_dropout_prob
     a__    =max_position_embeddings
     a__    =type_vocab_size
     a__    =type_sequence_label_size
     a__    =initializer_range
     a__    =num_choices



   def 					_UpperCAmelCase					(							self			)				->       List[Any]:






     """simple docstring"""






     a__    =ids_tensor([self.batch_size, self.seq_length],       self.vocab_size			)

     a__    =None
     if self.use_attention_mask:
       a__    =random_attention_mask([self.batch_size, self.seq_length]			)

     a__    =None
     if self.use_token_type_ids:
       a__    =ids_tensor([self.batch_size, self.seq_length],       self.type_vocab_size			)

     a__    =ElectraConfig(
         vocab_size=self.vocab_size,       hidden_size=self.hidden_size,       embedding_size=self.embedding_size,       num_hidden_layers=self.num_hidden_layers,       num_attention_heads=self.num_attention_heads,       intermediate_size=self.intermediate_size,       hidden_act=self.hidden_act,       hidden_dropout_prob=self.hidden_dropout_prob,       attention_probs_dropout_prob=self.attention_probs_dropout_prob,       max_position_embeddings=self.max_position_embeddings,       type_vocab_size=self.type_vocab_size,       initializer_range=self.initializer_range,       )

     return config, input_ids, token_type_ids, attention_mask



   def 					_UpperCAmelCase					(							self			)				->       List[Any]:






     """simple docstring"""






     a__    =self.prepare_config_and_inputs()
     a__, a__, a__, a__    =config_and_inputs
     a__    ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
     return config, inputs_dict







@require_flax
class 		__magic_name__     (				lowerCamelCase__						,			unittest.TestCase				):

   '''simple docstring'''




   lowerCamelCase__					:      List[str]        =  True

   lowerCamelCase__					:      Tuple        =  (
       (
           FlaxElectraModel,
           FlaxElectraForCausalLM,
           FlaxElectraForMaskedLM,
           FlaxElectraForPreTraining,
           FlaxElectraForTokenClassification,
           FlaxElectraForQuestionAnswering,
           FlaxElectraForMultipleChoice,
           FlaxElectraForSequenceClassification,
       )
       if is_flax_available()
       else ()
   )



   def 					_UpperCAmelCase					(							self			)				->       str:






     """simple docstring"""






     a__    =FlaxElectraModelTester(self			)



   @slow
   def 					_UpperCAmelCase					(							self			)				->       str:






     """simple docstring"""






     for model_class_name in self.all_model_classes:
       if model_class_name == FlaxElectraForMaskedLM:
         a__    =model_class_name.from_pretrained('''google/electra-small-generator'''			)
       else:
         a__    =model_class_name.from_pretrained('''google/electra-small-discriminator'''			)
       a__    =model(np.ones((1, 1)			)			)
       self.assertIsNotNone(lowercase_			)
