



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


UpperCamelCase_									=   logging.get_logger(__name__)

UpperCamelCase_									=   {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_									=   {
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''allenai/longformer-base-4096''': (
            '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_									=   {
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}







class       _snake_case				(     __snake_case ):






   '''simple docstring'''






   A__      : int       =						VOCAB_FILES_NAMES
   A__      : Dict       =						PRETRAINED_VOCAB_FILES_MAP
   A__      : Any       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__      : Optional[int]       =						["input_ids", "attention_mask"]
   A__      : List[Any]       =						LongformerTokenizer


   def __init__(    self:     List[str]  ,lowerCamelCase_:     Union[str, Any]=None  ,lowerCamelCase_:     Union[str, Any]=None  ,lowerCamelCase_:     str=None  ,lowerCamelCase_:     List[Any]="replace"  ,lowerCamelCase_:     Optional[int]="<s>"  ,lowerCamelCase_:     Any="</s>"  ,lowerCamelCase_:     Union[str, Any]="</s>"  ,lowerCamelCase_:     Dict="<s>"  ,lowerCamelCase_:     Tuple="<unk>"  ,lowerCamelCase_:     int="<pad>"  ,lowerCamelCase_:     int="<mask>"  ,lowerCamelCase_:     str=False  ,lowerCamelCase_:     List[str]=True  ,**lowerCamelCase_:     str  ,)	->      Union[str, Any]:
       super().__init__(
           lowerCamelCase_  ,lowerCamelCase_  ,tokenizer_file=lowerCamelCase_  ,errors=lowerCamelCase_  ,bos_token=lowerCamelCase_  ,eos_token=lowerCamelCase_  ,sep_token=lowerCamelCase_  ,cls_token=lowerCamelCase_  ,unk_token=lowerCamelCase_  ,pad_token=lowerCamelCase_  ,mask_token=lowerCamelCase_  ,add_prefix_space=lowerCamelCase_  ,trim_offsets=lowerCamelCase_  ,**lowerCamelCase_  ,)

       UpperCAmelCase_     :       Dict  		=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
       if pre_tok_state.get("""add_prefix_space"""  ,lowerCamelCase_							) != add_prefix_space:
           UpperCAmelCase_     :       Dict  		=					getattr(lowerCamelCase_  ,pre_tok_state.pop("""type"""							)							)
           UpperCAmelCase_     :       List[str]  		=					add_prefix_space
           UpperCAmelCase_     :       Any  		=					pre_tok_class(**lowerCamelCase_							)

       UpperCAmelCase_     :       Any  		=					add_prefix_space

       UpperCAmelCase_     :       Optional[int]  		=					"""post_processor"""
       UpperCAmelCase_     :       Tuple  		=					getattr(self.backend_tokenizer  ,lowerCamelCase_  ,lowerCamelCase_							)
       if tokenizer_component_instance:
           UpperCAmelCase_     :       Tuple  		=					json.loads(tokenizer_component_instance.__getstate__()							)

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
               UpperCAmelCase_     :       List[Any]  		=					tuple(state["""sep"""]							)
           if "cls" in state:
               UpperCAmelCase_     :       Union[str, Any]  		=					tuple(state["""cls"""]							)

           UpperCAmelCase_     :       Union[str, Any]  		=					False

           if state.get("""add_prefix_space"""  ,lowerCamelCase_							) != add_prefix_space:
               UpperCAmelCase_     :       List[Any]  		=					add_prefix_space
               UpperCAmelCase_     :       List[Any]  		=					True

           if state.get("""trim_offsets"""  ,lowerCamelCase_							) != trim_offsets:
               UpperCAmelCase_     :       Tuple  		=					trim_offsets
               UpperCAmelCase_     :       List[Any]  		=					True

           if changes_to_apply:
               UpperCAmelCase_     :       Union[str, Any]  		=					getattr(lowerCamelCase_  ,state.pop("""type"""							)							)
               UpperCAmelCase_     :       Optional[Any]  		=					component_class(**lowerCamelCase_							)
               setattr(self.backend_tokenizer  ,lowerCamelCase_  ,lowerCamelCase_							)


   @property
   def       A__    (    self:     Tuple							)	->      str:
       if self._mask_token is None:
           if self.verbose:
               logger.error("""Using mask_token, but it is not set yet."""							)
           return None
       return str(self._mask_token							)


   @mask_token.setter
   def       A__    (    self:     str  ,lowerCamelCase_:     List[Any]							)	->      List[str]:
       UpperCAmelCase_     :       Union[str, Any]  		=					AddedToken(lowerCamelCase_  ,lstrip=lowerCamelCase_  ,rstrip=lowerCamelCase_							) if isinstance(lowerCamelCase_  ,lowerCamelCase_							) else value
       UpperCAmelCase_     :       Tuple  		=					value


   def       A__    (    self:     int  ,*lowerCamelCase_:     int  ,**lowerCamelCase_:     Union[str, Any]							)	->      BatchEncoding:
       UpperCAmelCase_     :       List[str]  		=					kwargs.get("""is_split_into_words"""  ,lowerCamelCase_							)
       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*lowerCamelCase_  ,**lowerCamelCase_							)


   def       A__    (    self:     int  ,*lowerCamelCase_:     Union[str, Any]  ,**lowerCamelCase_:     str							)	->      BatchEncoding:
       UpperCAmelCase_     :       Optional[int]  		=					kwargs.get("""is_split_into_words"""  ,lowerCamelCase_							)

       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*lowerCamelCase_  ,**lowerCamelCase_							)


   def       A__    (    self:     Any  ,lowerCamelCase_:     str  ,lowerCamelCase_:     Optional[str] = None							)	->      Tuple[str]:
       UpperCAmelCase_     :       str  		=					self._tokenizer.model.save(lowerCamelCase_  ,name=lowerCamelCase_							)
       return tuple(lowerCamelCase_							)


   def       A__    (    self:     int  ,lowerCamelCase_:     Optional[Any]  ,lowerCamelCase_:     int=None							)	->      Optional[int]:
       UpperCAmelCase_     :       str  		=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
           return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






   def       A__    (    self:     Any  ,lowerCamelCase_:     List[int]  ,lowerCamelCase_:     Optional[List[int]] = None							)	->      List[int]:
       UpperCAmelCase_     :       Dict  		=					[self.sep_token_id]
       UpperCAmelCase_     :       Tuple  		=					[self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep							) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]







'''simple docstring'''


from typing import Callable




def       A							(      _UpperCAmelCase   :				Callable					)  ->				Dict:


  '''simple docstring'''




  def wrapper(_UpperCAmelCase   :				Optional[Any]					):
    __lowerCAmelCase		:    List[Any]			     =  function_with_docstring.__doc__
    return target_function

  return wrapper





