
'''simple docstring'''





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			UpperCamelCase__	(     SCREAMING_SNAKE_CASE   ,  unittest.TestCase):
 UpperCAmelCase__     :     int    					=       RealmTokenizer
 UpperCAmelCase__     :     Union[str, Any]    					=       RealmTokenizerFast
 UpperCAmelCase__     :     List[Any]    					=       True
 UpperCAmelCase__     :     Union[str, Any]    					=       True
 UpperCAmelCase__     :     Union[str, Any]    					=       filter_non_english






 def 						lowercase_    (      self	:str      )					->      Optional[int]:







       '''simple docstring'''



       super().setUp()

       __A													=      [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       __A													=      os.path.join(self.tmpdirname						,					VOCAB_FILES_NAMES['vocab_file']      )
       with open(self.vocab_file						,					'w'						,					encoding='utf-8'      ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )






 def 						lowercase_    (      self	:Any						,					_A	:Optional[Any]      )					->      List[str]:







       '''simple docstring'''



       __A													=      'UNwant\u00E9d,running'
       __A													=      'unwanted, running'
       return input_text, output_text






 def 						lowercase_    (      self	:str      )					->      List[Any]:







       '''simple docstring'''



       __A													=      self.tokenizer_class(self.vocab_file      )

       __A													=      tokenizer.tokenize('UNwant\u00E9d,running'      )
       self.assertListEqual(_A						,					['un', '##want', '##ed', ',', 'runn', '##ing']      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )						,					[9, 6, 7, 12, 10, 11]      )






 def 						lowercase_    (      self	:int      )					->      Any:







       '''simple docstring'''



       if not self.test_rust_tokenizer:
             return

       __A													=      self.get_tokenizer()
       __A													=      self.get_rust_tokenizer()

       __A													=      'UNwant\u00E9d,running'

       __A													=      tokenizer.tokenize(_A      )
       __A													=      rust_tokenizer.tokenize(_A      )
       self.assertListEqual(_A						,					_A      )

       __A													=      tokenizer.encode(_A						,					add_special_tokens=_A      )
       __A													=      rust_tokenizer.encode(_A						,					add_special_tokens=_A      )
       self.assertListEqual(_A						,					_A      )

       __A													=      self.get_rust_tokenizer()
       __A													=      tokenizer.encode(_A      )
       __A													=      rust_tokenizer.encode(_A      )
       self.assertListEqual(_A						,					_A      )

       # With lower casing
       __A													=      self.get_tokenizer(do_lower_case=_A      )
       __A													=      self.get_rust_tokenizer(do_lower_case=_A      )

       __A													=      'UNwant\u00E9d,running'

       __A													=      tokenizer.tokenize(_A      )
       __A													=      rust_tokenizer.tokenize(_A      )
       self.assertListEqual(_A						,					_A      )

       __A													=      tokenizer.encode(_A						,					add_special_tokens=_A      )
       __A													=      rust_tokenizer.encode(_A						,					add_special_tokens=_A      )
       self.assertListEqual(_A						,					_A      )

       __A													=      self.get_rust_tokenizer()
       __A													=      tokenizer.encode(_A      )
       __A													=      rust_tokenizer.encode(_A      )
       self.assertListEqual(_A						,					_A      )






 def 						lowercase_    (      self	:Tuple      )					->      List[Any]:







       '''simple docstring'''



       __A													=      BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'      )						,					['ah', '\u535A', '\u63A8', 'zz']      )






 def 						lowercase_    (      self	:Optional[int]      )					->      Optional[int]:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )						,					['hello', '!', 'how', 'are', 'you', '?']      )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,					['hello']      )






 def 						lowercase_    (      self	:Any      )					->      Tuple:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A						,					strip_accents=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,					['hällo', '!', 'how', 'are', 'you', '?']      )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,					['h\u00E9llo']      )






 def 						lowercase_    (      self	:int      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A						,					strip_accents=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,					['hallo', '!', 'how', 'are', 'you', '?']      )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,					['hello']      )






 def 						lowercase_    (      self	:Tuple      )					->      Any:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,					['hallo', '!', 'how', 'are', 'you', '?']      )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,					['hello']      )






 def 						lowercase_    (      self	:Optional[int]      )					->      int:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )						,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']      )






 def 						lowercase_    (      self	:str      )					->      Any:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A						,					strip_accents=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']      )






 def 						lowercase_    (      self	:List[Any]      )					->      Optional[int]:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A						,					strip_accents=_A      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']      )






 def 						lowercase_    (      self	:Any      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      BasicTokenizer(do_lower_case=_A						,					never_split=['[UNK]']      )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'      )						,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']      )






 def 						lowercase_    (      self	:Any      )					->      Optional[int]:







       '''simple docstring'''



       __A													=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       __A													=      {}
       for i, token in enumerate(_A      ):
             __A													=      i
       __A													=      WordpieceTokenizer(vocab=_A						,					unk_token='[UNK]'      )

       self.assertListEqual(tokenizer.tokenize(''      )						,					[]      )

       self.assertListEqual(tokenizer.tokenize('unwanted running'      )						,					['un', '##want', '##ed', 'runn', '##ing']      )

       self.assertListEqual(tokenizer.tokenize('unwantedX running'      )						,					['[UNK]', 'runn', '##ing']      )






 def 						lowercase_    (      self	:List[str]      )					->      Optional[Any]:







       '''simple docstring'''



       self.assertTrue(_is_whitespace(' '      )      )
       self.assertTrue(_is_whitespace('\t'      )      )
       self.assertTrue(_is_whitespace('\r'      )      )
       self.assertTrue(_is_whitespace('\n'      )      )
       self.assertTrue(_is_whitespace('\u00A0'      )      )

       self.assertFalse(_is_whitespace('A'      )      )
       self.assertFalse(_is_whitespace('-'      )      )






 def 						lowercase_    (      self	:int      )					->      Tuple:







       '''simple docstring'''



       self.assertTrue(_is_control('\u0005'      )      )

       self.assertFalse(_is_control('A'      )      )
       self.assertFalse(_is_control(' '      )      )
       self.assertFalse(_is_control('\t'      )      )
       self.assertFalse(_is_control('\r'      )      )






 def 						lowercase_    (      self	:Dict      )					->      Any:







       '''simple docstring'''



       self.assertTrue(_is_punctuation('-'      )      )
       self.assertTrue(_is_punctuation('$'      )      )
       self.assertTrue(_is_punctuation('`'      )      )
       self.assertTrue(_is_punctuation('.'      )      )

       self.assertFalse(_is_punctuation('A'      )      )
       self.assertFalse(_is_punctuation(' '      )      )






 def 						lowercase_    (      self	:int      )					->      Dict:







       '''simple docstring'''



       __A													=      self.get_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_A      ) for t in ['Test', '\xad', 'test']]						,					[['[UNK]'], [], ['[UNK]']]      )

       if self.test_rust_tokenizer:
             __A													=      self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(_A      ) for t in ['Test', '\xad', 'test']]						,					[['[UNK]'], [], ['[UNK]']]      )






 @slow
 def 						lowercase_    (      self	:int      )					->      List[Any]:







       '''simple docstring'''



       __A													=      self.tokenizer_class.from_pretrained('bert-base-uncased'      )

       __A													=      tokenizer.encode('sequence builders'						,					add_special_tokens=_A      )
       __A													=      tokenizer.encode('multi-sequence build'						,					add_special_tokens=_A      )

       __A													=      tokenizer.build_inputs_with_special_tokens(_A      )
       __A													=      tokenizer.build_inputs_with_special_tokens(_A						,					_A      )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]






 def 						lowercase_    (      self	:Optional[int]      )					->      str:







       '''simple docstring'''



       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                   __A													=      self.rust_tokenizer_class.from_pretrained(_A						,					**_A      )

                   __A													=      F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                   __A													=      tokenizer_r.encode_plus(
                       _A						,					return_attention_mask=_A						,					return_token_type_ids=_A						,					return_offsets_mapping=_A						,					add_special_tokens=_A						,					)

                   __A													=      tokenizer_r.do_lower_case if hasattr(_A						,					'do_lower_case'      ) else False
                   __A													=      (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]						,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']      )      )
                   self.assertEqual([e[0] for e in expected_results]						,					tokens['offset_mapping']      )






 @slow
 def 						lowercase_    (      self	:List[Any]      )					->      Tuple:







       '''simple docstring'''



       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                   __A													=      self.rust_tokenizer_class.from_pretrained(_A						,					**_A      )
                   __A													=      self.tokenizer_class.from_pretrained(_A						,					**_A      )
                   __A													=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                   __A													=      tokenizer_r.batch_encode_candidates(_A						,					max_length=10						,					return_tensors='np'      )
                   __A													=      tokenizer_p.batch_encode_candidates(_A						,					max_length=10						,					return_tensors='np'      )

                   __A													=      (2, 2, 10)

                   self.assertEqual(encoded_sentence_r['input_ids'].shape						,					_A      )
                   self.assertEqual(encoded_sentence_r['attention_mask'].shape						,					_A      )
                   self.assertEqual(encoded_sentence_r['token_type_ids'].shape						,					_A      )

                   self.assertEqual(encoded_sentence_p['input_ids'].shape						,					_A      )
                   self.assertEqual(encoded_sentence_p['attention_mask'].shape						,					_A      )
                   self.assertEqual(encoded_sentence_p['token_type_ids'].shape						,					_A      )








'''simple docstring'''





from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
 if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")):
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 from ...utils.dummy_torch_and_transformers_objects import (
     AudioLDMPipeline,
 )
else:
 from .pipeline_audioldm import AudioLDMPipeline






