



"""simple docstring"""






import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        __snake_case       (	A__  ,     unittest.TestCase):
  snake_case__				:   str  			=						LEDTokenizer
  snake_case__				:   List[str]  			=						LEDTokenizerFast
  snake_case__				:   str  			=						True




  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         super().setUp()
         _lowerCamelCase						:				Union[str, Any]    			= [
             "l",
             "o",
             "w",
             "e",
             "r",
             "s",
             "t",
             "i",
             "d",
             "n",
             "\u0120",
             "\u0120l",
             "\u0120n",
             "\u0120lo",
             "\u0120low",
             "er",
             "\u0120lowest",
             "\u0120newer",
             "\u0120wider",
             "<unk>",
         ]
         _lowerCamelCase						:				Optional[Any]    			= dict(zip(_A						,					range(len(_A		)		)		)		)
         _lowerCamelCase						:				Dict    			= ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
         _lowerCamelCase						:				Tuple    			= {"unk_token": "<unk>"}

         _lowerCamelCase						:				List[str]    			= os.path.join(self.tmpdirname						,					VOCAB_FILES_NAMES['''vocab_file''']		)
         _lowerCamelCase						:				Union[str, Any]    			= os.path.join(self.tmpdirname						,					VOCAB_FILES_NAMES['''merges_file''']		)
         with open(self.vocab_file						,					'''w'''						,					encoding='''utf-8'''		) as fp:
                fp.write(json.dumps(_A		) + '''\n'''		)
         with open(self.merges_file						,					'''w'''						,					encoding='''utf-8'''		) as fp:
                fp.write('''\n'''.join(_A		)		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple						,					**__lowerCAmelCase     :						List[str]		):






         """simple docstring"""






         kwargs.update(self.special_tokens_map		)
         return self.tokenizer_class.from_pretrained(self.tmpdirname						,					**_A		)




  def 						SCREAMING_SNAKE_CASE				( self     :						str						,					**__lowerCAmelCase     :						Union[str, Any]		):






         """simple docstring"""






         kwargs.update(self.special_tokens_map		)
         return self.rust_tokenizer_class.from_pretrained(self.tmpdirname						,					**_A		)




  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple						,					__lowerCAmelCase     :						Any		):






         """simple docstring"""






         return "lower newer", "lower newer"




  @cached_property
  def 						SCREAMING_SNAKE_CASE				( self     :						Tuple		):






         """simple docstring"""






         return LEDTokenizer.from_pretrained('''allenai/led-base-16384'''		)




  @cached_property
  def 						SCREAMING_SNAKE_CASE				( self     :						List[Any]		):






         """simple docstring"""






         return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384'''		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Optional[int]    			= ["A long paragraph for summarization.", "Another paragraph for summarization."]
         _lowerCamelCase						:				List[Any]    			= [0, 2_5_0, 2_5_1, 1_7_8_1_8, 1_3, 3_9_1_8_6, 1_9_3_8, 4, 2]

         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				Optional[Any]    			= tokenizer(_A						,					max_length=len(_A		)						,					padding=_A						,					return_tensors='''pt'''		)
                self.assertIsInstance(_A						,					_A		)

                self.assertEqual((2, 9)						,					batch.input_ids.shape		)
                self.assertEqual((2, 9)						,					batch.attention_mask.shape		)
                _lowerCamelCase						:				List[str]    			= batch.input_ids.tolist()[0]
                self.assertListEqual(_A						,					_A		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						List[str]		):






         """simple docstring"""






         _lowerCamelCase						:				str    			= ["A long paragraph for summarization.", "Another paragraph for summarization."]
         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				int    			= tokenizer(_A						,					padding=_A						,					return_tensors='''pt'''		)
                self.assertIn('''input_ids'''						,					_A		)
                self.assertIn('''attention_mask'''						,					_A		)
                self.assertNotIn('''labels'''						,					_A		)
                self.assertNotIn('''decoder_attention_mask'''						,					_A		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Dict    			= [
             "Summary of the text.",
             "Another summary.",
         ]
         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				Union[str, Any]    			= tokenizer(text_target=_A						,					max_length=3_2						,					padding='''max_length'''						,					return_tensors='''pt'''		)
                self.assertEqual(3_2						,					targets['''input_ids'''].shape[1]		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				List[str]    			= tokenizer(
                    ['''I am a small frog''' * 1_0_2_4, '''I am a small frog''']						,					padding=_A						,					truncation=_A						,					return_tensors='''pt'''		)
                self.assertIsInstance(_A						,					_A		)
                self.assertEqual(batch.input_ids.shape						,					(2, 5_1_2_2)		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         _lowerCamelCase						:				Tuple    			= ["A long paragraph for summarization."]
         _lowerCamelCase						:				str    			= [
             "Summary of the text.",
         ]
         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				Optional[Any]    			= tokenizer(_A						,					return_tensors='''pt'''		)
                _lowerCamelCase						:				Optional[int]    			= tokenizer(text_target=_A						,					return_tensors='''pt'''		)
                _lowerCamelCase						:				Optional[int]    			= inputs["input_ids"]
                _lowerCamelCase						:				Any    			= targets["input_ids"]
                self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()		)
                self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()		)
                self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()		)
                self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()		)




  @require_torch
  def 						SCREAMING_SNAKE_CASE				( self     :						Optional[Any]		):






         """simple docstring"""






         for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                _lowerCamelCase						:				str    			= ["Summary of the text.", "Another summary."]
                _lowerCamelCase						:				Optional[int]    			= [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                _lowerCamelCase						:				Optional[int]    			= tokenizer(_A						,					padding=_A		)
                _lowerCamelCase						:				List[Any]    			= [[0] * len(_A		) for x in encoded_output["input_ids"]]
                _lowerCamelCase						:				Optional[Any]    			= tokenizer.pad(_A		)
                self.assertSequenceEqual(outputs['''global_attention_mask''']						,					_A		)




  def 						SCREAMING_SNAKE_CASE				( self     :						List[Any]		):






         """simple docstring"""






         pass




  def 						SCREAMING_SNAKE_CASE				( self     :						str		):






         """simple docstring"""






         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
                       _lowerCamelCase						:				Optional[int]    			= self.rust_tokenizer_class.from_pretrained(_A						,					**_A		)
                       _lowerCamelCase						:				Dict    			= self.tokenizer_class.from_pretrained(_A						,					**_A		)
                       _lowerCamelCase						:				Any    			= "A, <mask> AllenNLP sentence."
                       _lowerCamelCase						:				Tuple    			= tokenizer_r.encode_plus(_A						,					add_special_tokens=_A						,					return_token_type_ids=_A		)
                       _lowerCamelCase						:				Any    			= tokenizer_p.encode_plus(_A						,					add_special_tokens=_A						,					return_token_type_ids=_A		)
                       self.assertEqual(sum(tokens_r['''token_type_ids''']		)						,					sum(tokens_p['''token_type_ids''']		)		)
                       self.assertEqual(
                           sum(tokens_r['''attention_mask''']		) / len(tokens_r['''attention_mask''']		)						,					sum(tokens_p['''attention_mask''']		) / len(tokens_p['''attention_mask''']		)						,					)

                       _lowerCamelCase						:				Dict    			= tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']		)
                       _lowerCamelCase						:				str    			= tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']		)
                       self.assertSequenceEqual(tokens_p['''input_ids''']						,					[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]		)
                       self.assertSequenceEqual(tokens_r['''input_ids''']						,					[0, 2_5_0, 6, 5_0_2_6_4, 3_8_2_3, 4_8_7, 2_1_9_9_2, 3_6_4_5, 4, 2]		)

                       self.assertSequenceEqual(
                           _A						,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']		)
                       self.assertSequenceEqual(
                           _A						,					['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']		)



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__lowerCamelCase					:	Optional[Any]       =  logging.get_logger(__name__)

__lowerCamelCase					:	Dict       =  {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase					:	int       =  {
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

__lowerCamelCase					:	Any       =  {
    '''junnyu/roformer_chinese_small''': 15_36,
    '''junnyu/roformer_chinese_base''': 15_36,
    '''junnyu/roformer_chinese_char_small''': 5_12,
    '''junnyu/roformer_chinese_char_base''': 5_12,
    '''junnyu/roformer_small_discriminator''': 1_28,
    '''junnyu/roformer_small_generator''': 1_28,
}


__lowerCamelCase					:	Union[str, Any]       =  {
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}






class 		a__						(					A__					):
					A	       =							VOCAB_FILES_NAMES
					A	       =							PRETRAINED_VOCAB_FILES_MAP
					A	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A	       =							PRETRAINED_INIT_CONFIGURATION
					A	       =							RoFormerTokenizer

					def __init__(    self      :       List[str],_A      :       int=None,_A      :       int=None,_A      :       int=True,_A      :       List[Any]="[UNK]",_A      :       Tuple="[SEP]",_A      :       List[Any]="[PAD]",_A      :       Optional[int]="[CLS]",_A      :       Optional[Any]="[MASK]",_A      :       Optional[int]=True,_A      :       List[str]=None,**_A      :       List[Any],):







							"""simple docstring"""






							super().__init__(
							    _A,tokenizer_file=_A,do_lower_case=_A,unk_token=_A,sep_token=_A,pad_token=_A,cls_token=_A,mask_token=_A,tokenize_chinese_chars=_A,strip_accents=_A,**_A,)

							SCREAMING_SNAKE_CASE_  :		Tuple				      =    json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
							if (
							    pre_tok_state.get("lowercase",_A    ) != do_lower_case
							    or pre_tok_state.get("strip_accents",_A    ) != strip_accents
							):
									SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    getattr(_A,pre_tok_state.pop("type"    )    )
									SCREAMING_SNAKE_CASE_  :		Any				      =    do_lower_case
									SCREAMING_SNAKE_CASE_  :		List[str]				      =    strip_accents
									SCREAMING_SNAKE_CASE_  :		str				      =    pre_tok_class(**_A    )

							SCREAMING_SNAKE_CASE_  :		List[str]				      =    do_lower_case

					def __getstate__(    self      :       Optional[int]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    self.__dict__.copy()
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    BertPreTokenizer()
							return state

					def __setstate__(    self      :       List[Any],_A      :       Union[str, Any]    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    d
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    self.__dict__["_tokenizer"].get_vocab()
							SCREAMING_SNAKE_CASE_  :		Any				      =    PreTokenizer.custom(JiebaPreTokenizer(_A    )    )

					def 				__UpperCamelCase   (    self      :       Union[str, Any],_A      :       List[Any],_A      :       str=None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output

					def 				__UpperCamelCase   (    self      :       str,_A      :       List[int],_A      :       Optional[List[int]] = None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    [self.sep_token_id]
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    [self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep    ) * [0]
							return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

					def 				__UpperCamelCase   (    self      :       int,_A      :       str,_A      :       Optional[str] = None    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    self._tokenizer.model.save(_A,name=_A    )
							return tuple(_A    )






					def 				__UpperCamelCase   (    self      :       int,_A      :       Optional[int],_A      :       List[Any]=None,_A      :       Tuple=None,_A      :       str=False,**_A      :       List[Any],):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    BertPreTokenizer()
							return super().save_pretrained(_A,_A,_A,_A,**_A    )
