



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


A_				:					Any							=      logging.get_logger(__name__)

A_				:					Union[str, Any]							=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A_				:					Optional[Any]							=      {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

A_				:					int							=      {
    """allenai/longformer-base-4096""": 4_0_9_6,
    """allenai/longformer-large-4096""": 4_0_9_6,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4_0_9_6,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4_0_9_6,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4_0_9_6,
}



class 				lowercase			(    _lowerCamelCase							):


 """simple docstring"""


 UpperCAmelCase		      =    VOCAB_FILES_NAMES
 UpperCAmelCase		      =    PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase		      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase		      =    ["""input_ids""", """attention_mask"""]
 UpperCAmelCase		      =    LongformerTokenizer







 def __init__(				self    ,a_=None    ,a_=None    ,a_=None    ,a_="replace"    ,a_="<s>"    ,a_="</s>"    ,a_="</s>"    ,a_="<s>"    ,a_="<unk>"    ,a_="<pad>"    ,a_="<mask>"    ,a_=False    ,a_=True    ,**a_    ,)						->  str:
        super().__init__(
            a_    ,a_    ,tokenizer_file=a_    ,errors=a_    ,bos_token=a_    ,eos_token=a_    ,sep_token=a_    ,cls_token=a_    ,unk_token=a_    ,pad_token=a_    ,mask_token=a_    ,add_prefix_space=a_    ,trim_offsets=a_    ,**a_    ,)

        _UpperCAmelCase    :		Any     	=					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
        if pre_tok_state.get("""add_prefix_space"""    ,a_				) != add_prefix_space:
               _UpperCAmelCase    :		List[str]     	=					getattr(a_    ,pre_tok_state.pop("""type"""				)				)
               _UpperCAmelCase    :		List[str]     	=					add_prefix_space
               _UpperCAmelCase    :		int     	=					pre_tok_class(**a_				)

        _UpperCAmelCase    :		Union[str, Any]     	=					add_prefix_space

        _UpperCAmelCase    :		Union[str, Any]     	=					"""post_processor"""
        _UpperCAmelCase    :		Optional[Any]     	=					getattr(self.backend_tokenizer    ,a_    ,a_				)
        if tokenizer_component_instance:
               _UpperCAmelCase    :		Tuple     	=					json.loads(tokenizer_component_instance.__getstate__()				)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                      _UpperCAmelCase    :		str     	=					tuple(state["""sep"""]				)
               if "cls" in state:
                      _UpperCAmelCase    :		str     	=					tuple(state["""cls"""]				)

               _UpperCAmelCase    :		int     	=					False

               if state.get("""add_prefix_space"""    ,a_				) != add_prefix_space:
                      _UpperCAmelCase    :		Dict     	=					add_prefix_space
                      _UpperCAmelCase    :		Optional[int]     	=					True

               if state.get("""trim_offsets"""    ,a_				) != trim_offsets:
                      _UpperCAmelCase    :		Optional[Any]     	=					trim_offsets
                      _UpperCAmelCase    :		str     	=					True

               if changes_to_apply:
                      _UpperCAmelCase    :		Dict     	=					getattr(a_    ,state.pop("""type"""				)				)
                      _UpperCAmelCase    :		Dict     	=					component_class(**a_				)
                      setattr(self.backend_tokenizer    ,a_    ,a_				)







 @property
 def _snake_case					(				self				)						->  str:
        if self._mask_token is None:
               if self.verbose:
                      logger.error("""Using mask_token, but it is not set yet."""				)
               return None
        return str(self._mask_token				)







 @mask_token.setter
 def _snake_case					(				self    ,a_				)						->  List[str]:
        _UpperCAmelCase    :		Tuple     	=					AddedToken(a_    ,lstrip=a_    ,rstrip=a_				) if isinstance(a_    ,a_				) else value
        _UpperCAmelCase    :		List[Any]     	=					value







 def _snake_case					(				self    ,*a_    ,**a_				)						->  BatchEncoding:
        _UpperCAmelCase    :		int     	=					kwargs.get("""is_split_into_words"""    ,a_				)
        assert self.add_prefix_space or not is_split_into_words, (
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*a_    ,**a_				)







 def _snake_case					(				self    ,*a_    ,**a_				)						->  BatchEncoding:
        _UpperCAmelCase    :		int     	=					kwargs.get("""is_split_into_words"""    ,a_				)

        assert self.add_prefix_space or not is_split_into_words, (
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*a_    ,**a_				)







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  Tuple[str]:
        _UpperCAmelCase    :		Any     	=					self._tokenizer.model.save(a_    ,name=a_				)
        return tuple(a_				)







 def _snake_case					(				self    ,a_    ,a_=None				)						->  Optional[int]:
        _UpperCAmelCase    :		str     	=					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
               return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







 def _snake_case					(				self    ,a_    ,a_ = None				)						->  List[int]:
        _UpperCAmelCase    :		Dict     	=					[self.sep_token_id]
        _UpperCAmelCase    :		Dict     	=					[self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]










"""simple docstring"""


import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
       SpeechaTextaDecoder,
       SpeechaTextaForCausalLM,
   )



@require_torch
class   lowercase_					:

       def __init__(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE=99 ,       __SCREAMING_SNAKE_CASE=13 ,       __SCREAMING_SNAKE_CASE=16 ,       __SCREAMING_SNAKE_CASE=7 ,       __SCREAMING_SNAKE_CASE=True ,       __SCREAMING_SNAKE_CASE=True ,       __SCREAMING_SNAKE_CASE=True ,       __SCREAMING_SNAKE_CASE=False ,       __SCREAMING_SNAKE_CASE=True ,       __SCREAMING_SNAKE_CASE=2 ,       __SCREAMING_SNAKE_CASE=32 ,       __SCREAMING_SNAKE_CASE=4 ,       __SCREAMING_SNAKE_CASE=4 ,       __SCREAMING_SNAKE_CASE=30 ,       __SCREAMING_SNAKE_CASE=0 ,       __SCREAMING_SNAKE_CASE=1 ,       __SCREAMING_SNAKE_CASE=2 ,       __SCREAMING_SNAKE_CASE=None ,       ):






        '''simple docstring'''

        snake_case__     				=					parent
        snake_case__     				=					batch_size
        snake_case__     				=					decoder_seq_length
        # For common tests
        snake_case__     				=					self.decoder_seq_length
        snake_case__     				=					is_training
        snake_case__     				=					use_attention_mask
        snake_case__     				=					use_labels

        snake_case__     				=					vocab_size
        snake_case__     				=					d_model
        snake_case__     				=					d_model
        snake_case__     				=					decoder_layers
        snake_case__     				=					decoder_layers
        snake_case__     				=					decoder_ffn_dim
        snake_case__     				=					decoder_attention_heads
        snake_case__     				=					decoder_attention_heads
        snake_case__     				=					eos_token_id
        snake_case__     				=					bos_token_id
        snake_case__     				=					pad_token_id
        snake_case__     				=					decoder_start_token_id
        snake_case__     				=					use_cache
        snake_case__     				=					max_position_embeddings

        snake_case__     				=					None
        snake_case__     				=					decoder_seq_length
        snake_case__     				=					2
        snake_case__     				=					1

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        snake_case__     				=					ids_tensor([self.batch_size, self.decoder_seq_length] ,       self.vocab_size  )

        snake_case__     				=					None
        if self.use_attention_mask:
         snake_case__     				=					ids_tensor([self.batch_size, self.decoder_seq_length] ,       vocab_size=2  )

        snake_case__     				=					None
        if self.use_labels:
         snake_case__     				=					ids_tensor([self.batch_size, self.decoder_seq_length] ,       self.vocab_size  )

        snake_case__     				=					SpeechaTextaConfig(
            vocab_size=self.vocab_size ,       d_model=self.d_model ,       decoder_layers=self.decoder_layers ,       decoder_ffn_dim=self.decoder_ffn_dim ,       decoder_attention_heads=self.decoder_attention_heads ,       eos_token_id=self.eos_token_id ,       bos_token_id=self.bos_token_id ,       use_cache=self.use_cache ,       pad_token_id=self.pad_token_id ,       decoder_start_token_id=self.decoder_start_token_id ,       max_position_embeddings=self.max_position_embeddings ,       )

        return (
            config,
            input_ids,
            attention_mask,
            lm_labels,
        )

       def 		_SCREAMING_SNAKE_CASE(      self ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       ):






        '''simple docstring'''

        snake_case__     				=					True
        snake_case__     				=					SpeechaTextaDecoder(config=__SCREAMING_SNAKE_CASE  ).to(__SCREAMING_SNAKE_CASE  ).eval()
        snake_case__     				=					input_ids[:2]

        input_ids[input_ids == 0] += 1
        # first forward pass
        snake_case__     				=					model(__SCREAMING_SNAKE_CASE ,       use_cache=__SCREAMING_SNAKE_CASE  )
        snake_case__     				=					model(__SCREAMING_SNAKE_CASE  )
        snake_case__     				=					model(__SCREAMING_SNAKE_CASE ,       use_cache=__SCREAMING_SNAKE_CASE  )

        self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE  ) == len(__SCREAMING_SNAKE_CASE  )  )
        self.parent.assertTrue(len(__SCREAMING_SNAKE_CASE  ) == len(__SCREAMING_SNAKE_CASE  ) + 1  )

        snake_case__     				=					outputs["past_key_values"]

        # create hypothetical next token and extent to next_input_ids
        snake_case__     				=					ids_tensor((2, 1) ,       config.vocab_size - 1  ) + 1

        # append to next input_ids and
        snake_case__     				=					torch.cat([input_ids, next_tokens] ,       dim=-1  )

        print(__SCREAMING_SNAKE_CASE  )

        snake_case__     				=					model(__SCREAMING_SNAKE_CASE  )["last_hidden_state"]
        snake_case__     				=					model(__SCREAMING_SNAKE_CASE ,       past_key_values=__SCREAMING_SNAKE_CASE  )["last_hidden_state"]

        # select random slice
        snake_case__     				=					ids_tensor((1,) ,       output_from_past.shape[-1]  ).item()
        snake_case__     				=					output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
        snake_case__     				=					output_from_past[:, 0, random_slice_idx].detach()

        # test that outputs are equal for slice
        assert torch.allclose(__SCREAMING_SNAKE_CASE ,       __SCREAMING_SNAKE_CASE ,       atol=1e-3  )

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        snake_case__     				=					self.prepare_config_and_inputs()
        (
            (
            snake_case__
        )				, (
            snake_case__
        )				, (
            snake_case__
        )				, (
            snake_case__
        )				, 
        )     				=					config_and_inputs

        snake_case__     				=					{
            "input_ids": input_ids,
            "attention_mask": attention_mask,
        }
        return config, inputs_dict



@require_torch
class   lowercase_					(  __lowerCAmelCase					,__lowerCAmelCase					,__lowerCAmelCase					,unittest.TestCase				):
       snake_case__					:				Tuple									=  (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
       snake_case__					:				Optional[Any]									=  (SpeechaTextaForCausalLM,) if is_torch_available() else ()
       snake_case__					:				List[Any]									=  {"""text-generation""": SpeechaTextaForCausalLM} if is_torch_available() else {}
       snake_case__					:				Dict									=  True
       snake_case__					:				Dict									=  False

       def 		_SCREAMING_SNAKE_CASE(      self ,       ):






        '''simple docstring'''

        snake_case__     				=					SpeechaTextaStandaloneDecoderModelTester(self ,       is_training=__SCREAMING_SNAKE_CASE  )
        snake_case__     				=					ConfigTester(self ,       config_class=__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        pass

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        pass

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        pass

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        self.config_tester.run_common_tests()

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        snake_case__     				=					self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past(*__SCREAMING_SNAKE_CASE  )

       def 		_SCREAMING_SNAKE_CASE(      self  ):






        '''simple docstring'''

        return

