

'''simple docstring'''
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase :      Tuple					 =     get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
     import sentencepiece as sp


lowerCamelCase :      List[str]					 =     5
lowerCamelCase :      List[Any]					 =     1_0







@require_sentencepiece
@require_tokenizers
class 							A__    (       A__     ,   unittest.TestCase  ):
  A__		 =				SpeechaTextTokenizer
  A__		 =				False
  A__		 =				True





  def 						A  (		self      :	str )	->    List[str]:





       '''simple docstring'''






       super().setUp()

       _SCREAMING_SNAKE_CASE							    =sp.SentencePieceProcessor()
       spm_model.Load(_a )
       _SCREAMING_SNAKE_CASE							    =['<s>', '<pad>', '</s>', '<unk>']

       vocab += [spm_model.IdToPiece(id_ ) for id_ in range(len(_a ) )]
       _SCREAMING_SNAKE_CASE							    =dict(zip(_a							,					range(len(_a ) ) ) )

       _SCREAMING_SNAKE_CASE							    =Path(self.tmpdirname )
       save_json(_a							,					save_dir / VOCAB_FILES_NAMES['vocab_file'] )
       if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
            copyfile(_a							,					save_dir / VOCAB_FILES_NAMES['spm_file'] )

       _SCREAMING_SNAKE_CASE							    =SpeechaTextTokenizer.from_pretrained(self.tmpdirname )
       tokenizer.save_pretrained(self.tmpdirname )





  def 						A  (		self      :	Union[str, Any] )	->    Tuple:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='<pad>'
       _SCREAMING_SNAKE_CASE							    =1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a )							,					_a )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a )							,					_a )





  def 						A  (		self      :	List[str] )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =list(self.get_tokenizer().get_vocab().keys() )

       self.assertEqual(vocab_keys[0]							,					'<s>' )
       self.assertEqual(vocab_keys[1]							,					'<pad>' )
       self.assertEqual(vocab_keys[-1]							,					'j' )
       self.assertEqual(len(_a )							,					1001 )





  def 						A  (		self      :	Dict )	->    Union[str, Any]:





       '''simple docstring'''






       self.assertEqual(self.get_tokenizer().vocab_size							,					1001 )





  def 						A  (		self      :	List[Any] )	->    Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =SpeechaTextTokenizer.from_pretrained(self.tmpdirname )

       _SCREAMING_SNAKE_CASE							    =tokenizer.tokenize('This is a test' )
       self.assertListEqual(_a							,					['▁This', '▁is', '▁a', '▁t', 'est'] )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_a )							,					[289, 50, 14, 174, 386]							,					)

       _SCREAMING_SNAKE_CASE							    =tokenizer.tokenize('I was born in 92000, and this is falsé.' )
       self.assertListEqual(
           _a							,					[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']							,					)
       _SCREAMING_SNAKE_CASE							    =tokenizer.convert_tokens_to_ids(_a )
       self.assertListEqual(_a							,					[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8] )

       _SCREAMING_SNAKE_CASE							    =tokenizer.convert_ids_to_tokens(_a )
       self.assertListEqual(
           _a							,					[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']							,					)





  @slow
  def 						A  (		self      :	Any )	->    str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ={'input_ids': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_a							,					model_name='facebook/s2t-small-mustc-en-de-st'							,					revision='a14f04cf0776c02f62a8cb800cf7909e15ea23ad'							,					)







@require_sentencepiece
class 							A__    (       unittest.TestCase  ):
  A__		 =				'valhalla/s2t_mustc_multilinguial_medium'

  A__		 =				'C\'est trop cool'
  A__		 =				'Esto es genial'





  @classmethod
  def 						A  (		cls      :	Tuple )	->    Tuple:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name )
       return cls





  def 						A  (		self      :	int )	->    List[str]:





       '''simple docstring'''






       self.assertEqual(self.tokenizer.lang_code_to_id['pt']							,					4 )
       self.assertEqual(self.tokenizer.lang_code_to_id['ru']							,					6 )
       self.assertEqual(self.tokenizer.lang_code_to_id['it']							,					9 )
       self.assertEqual(self.tokenizer.lang_code_to_id['de']							,					11 )





  def 						A  (		self      :	str )	->    Tuple:





       '''simple docstring'''






       self.assertEqual(self.tokenizer.vocab_size							,					1_0000 )





  def 						A  (		self      :	List[Any] )	->    Optional[Any]:





       '''simple docstring'''






       self.assertIn(_a							,					self.tokenizer.all_special_ids )
       _SCREAMING_SNAKE_CASE							    =[ES_CODE, 4, 1601, 47, 7647, 2]
       _SCREAMING_SNAKE_CASE							    =self.tokenizer.decode(_a							,					skip_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =self.tokenizer.decode(generated_ids[1:]							,					skip_special_tokens=_a )
       self.assertEqual(_a							,					_a )
       self.assertNotIn(self.tokenizer.eos_token							,					_a )





  def 						A  (		self      :	Tuple )	->    Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='fr'
       _SCREAMING_SNAKE_CASE							    =self.tokenizer(self.french_text ).input_ids
       self.assertEqual(encoded[0]							,					_a )
       self.assertEqual(encoded[-1]							,					self.tokenizer.eos_token_id )





  def 						A  (		self      :	Any )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    ='fr'
       self.assertListEqual(self.tokenizer.prefix_tokens							,					[FR_CODE] )

       _SCREAMING_SNAKE_CASE							    ='es'
       self.assertListEqual(self.tokenizer.prefix_tokens							,					[ES_CODE] )



'''simple docstring'''
class 							A__    :





  def __init__(		self      :	Union[str, Any]							,					_a      :	int )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =size
       _SCREAMING_SNAKE_CASE							    =[0] * size
       _SCREAMING_SNAKE_CASE							    =[0] * size





  @staticmethod
  def 						A  (		_a      :	int )	->    int:





       '''simple docstring'''






       return index | (index + 1)





  @staticmethod
  def 						A  (		_a      :	int )	->    int:





       '''simple docstring'''






       return (index & (index + 1)) - 1





  def 						A  (		self      :	Tuple							,					_a      :	int							,					_a      :	int )	->    None:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =value
       while index < self.size:
            _SCREAMING_SNAKE_CASE							    =self.get_prev(_a ) + 1
            if current_left_border == index:
                 _SCREAMING_SNAKE_CASE							    =value
            else:
                 _SCREAMING_SNAKE_CASE							    =max(_a							,					_a							,					_a )
            _SCREAMING_SNAKE_CASE							    =self.get_next(_a )





  def 						A  (		self      :	int							,					_a      :	int							,					_a      :	int )	->    int:





       '''simple docstring'''






       right -= 1  # Because of right is exclusive
       _SCREAMING_SNAKE_CASE							    =0
       while left <= right:
            _SCREAMING_SNAKE_CASE							    =self.get_prev(_a )
            if left <= current_left:
                 _SCREAMING_SNAKE_CASE							    =max(_a							,					self.tree[right] )
                 _SCREAMING_SNAKE_CASE							    =current_left
            else:
                 _SCREAMING_SNAKE_CASE							    =max(_a							,					self.arr[right] )
                 right -= 1
       return result


if __name__ == "__main__":
     import doctest

     doctest.testmod()
