




'''simple docstring'''


import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						lowerCAmelCase_	(       UpperCAmelCase_       ,					unittest.TestCase							):







  '''simple docstring'''

  UpperCamelCase_			:      Any										=				HerbertTokenizer
  UpperCamelCase_			:      List[str]										=				HerbertTokenizerFast
  UpperCamelCase_			:      Union[str, Any]										=				True



  def 					_snake_case     ( self				:							List[Any]	)			->		int:







       '''simple docstring'''



       super().setUp()

       # Use a simpler test file without japanese/chinese characters
       with open(f"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"""					,  encoding='''utf-8'''	) as f_data:
            A:		Optional[int]          =		f_data.read().replace('''\n\n'''					,  '''\n'''	).strip()

       A:		List[str]          =		[
           '''<s>''',
           '''</s>''',
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''w</w>''',
           '''r</w>''',
           '''t</w>''',
           '''lo''',
           '''low''',
           '''er</w>''',
           '''low</w>''',
           '''lowest</w>''',
           '''newer</w>''',
           '''wider</w>''',
           ''',</w>''',
           '''<unk>''',
       ]
       A:		Tuple          =		dict(zip(SCREAMING_SNAKE_CASE_					,  range(len(SCREAMING_SNAKE_CASE_	)	)	)	)
       A:		Optional[Any]          =		['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

       A:		List[str]          =		os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['''vocab_file''']	)
       A:		Optional[Any]          =		os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['''merges_file''']	)
       with open(self.vocab_file					,  '''w'''	) as fp:
            fp.write(json.dumps(SCREAMING_SNAKE_CASE_	)	)
       with open(self.merges_file					,  '''w'''	) as fp:
            fp.write('''\n'''.join(SCREAMING_SNAKE_CASE_	)	)



  def 					_snake_case     ( self				:							Optional[int]					,  SCREAMING_SNAKE_CASE_				:							str	)			->		List[Any]:







       '''simple docstring'''



       A:		Dict          =		'''lower newer'''
       A:		int          =		'''lower newer'''
       return input_text, output_text



  def 					_snake_case     ( self				:							int	)			->		Dict:







       '''simple docstring'''



       A:		List[str]          =		self.tokenizer_class(vocab_file=self.vocab_file					,  merges_file=self.merges_file	)

       A:		Any          =		'''lower'''
       A:		int          =		['''low''', '''er</w>''']
       A:		str          =		tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

       A:		Optional[int]          =		tokens + ['''<unk>''']
       A:		str          =		[16, 17, 23]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)					,  SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							str	)			->		int:







       '''simple docstring'''



       if not self.test_rust_tokenizer:
            return

       A:		List[str]          =		self.get_tokenizer()
       A:		List[Any]          =		self.get_rust_tokenizer()

       A:		Dict          =		'''lower,newer'''

       A:		Tuple          =		tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
       A:		Dict          =		rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

       A:		Dict          =		tokenizer.encode(SCREAMING_SNAKE_CASE_					,  add_special_tokens=SCREAMING_SNAKE_CASE_	)
       A:		Dict          =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE_					,  add_special_tokens=SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

       A:		Any          =		self.get_rust_tokenizer()
       A:		Dict          =		tokenizer.encode(SCREAMING_SNAKE_CASE_	)
       A:		Optional[Any]          =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE_	)
       self.assertListEqual(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)




  @slow
  def 					_snake_case     ( self				:							Any	)			->		Optional[int]:







       '''simple docstring'''



       A:		Optional[int]          =		self.tokenizer_class.from_pretrained('''allegro/herbert-base-cased'''	)

       A:		Dict          =		tokenizer.encode('''konstruowanie sekwencji'''					,  add_special_tokens=SCREAMING_SNAKE_CASE_	)
       A:		Optional[Any]          =		tokenizer.encode('''konstruowanie wielu sekwencji'''					,  add_special_tokens=SCREAMING_SNAKE_CASE_	)

       A:		str          =		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_	)
       A:		Union[str, Any]          =		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)

       assert encoded_sentence == [0] + text + [2]
       assert encoded_pair == [0] + text + [2] + text_a + [2]












'''simple docstring'''


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class 						lowerCAmelCase_	(       UpperCAmelCase_							):







  '''simple docstring'''

  UpperCamelCase_			:      Optional[int]										=				"""AutoFeatureExtractor"""
  UpperCamelCase_			:      List[Any]										=				"""Speech2Text2Tokenizer"""



  def __init__( self				:							str					,  SCREAMING_SNAKE_CASE_				:							str					,  SCREAMING_SNAKE_CASE_				:							int	)			->		Any:







       '''simple docstring'''



       super().__init__(SCREAMING_SNAKE_CASE_					,  SCREAMING_SNAKE_CASE_	)
       A:		Union[str, Any]          =		self.feature_extractor
       A:		Dict          =		False



  def __call__( self				:							Any					,  *SCREAMING_SNAKE_CASE_				:							Any					,  **SCREAMING_SNAKE_CASE_				:							Optional[int]	)			->		Optional[Any]:







       '''simple docstring'''



       if self._in_target_context_manager:
            return self.current_processor(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)

       if "raw_speech" in kwargs:
            warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''	)
            A:		List[str]          =		kwargs.pop('''raw_speech'''	)
       else:
            A:		Any          =		kwargs.pop('''audio'''					,  SCREAMING_SNAKE_CASE_	)
       A:		List[Any]          =		kwargs.pop('''sampling_rate'''					,  SCREAMING_SNAKE_CASE_	)
       A:		Optional[int]          =		kwargs.pop('''text'''					,  SCREAMING_SNAKE_CASE_	)
       if len(SCREAMING_SNAKE_CASE_	) > 0:
            A:		str          =		args[0]
            A:		Tuple          =		args[1:]

       if audio is None and text is None:
            raise ValueError('''You need to specify either an `audio` or `text` input to process.'''	)

       if audio is not None:
            A:		Tuple          =		self.feature_extractor(SCREAMING_SNAKE_CASE_					,  *SCREAMING_SNAKE_CASE_					,  sampling_rate=SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)
       if text is not None:
            A:		List[Any]          =		self.tokenizer(SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)

       if text is None:
            return inputs
       elif audio is None:
            return encodings
       else:
            A:		str          =		encodings['''input_ids''']
            return inputs



  def 					_snake_case     ( self				:							Tuple					,  *SCREAMING_SNAKE_CASE_				:							int					,  **SCREAMING_SNAKE_CASE_				:							Union[str, Any]	)			->		str:







       '''simple docstring'''



       return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)



  def 					_snake_case     ( self				:							Optional[int]					,  *SCREAMING_SNAKE_CASE_				:							str					,  **SCREAMING_SNAKE_CASE_				:							int	)			->		Union[str, Any]:







       '''simple docstring'''



       return self.tokenizer.decode(*SCREAMING_SNAKE_CASE_					,  **SCREAMING_SNAKE_CASE_	)




  @contextmanager
  def 					_snake_case     ( self				:							str	)			->		int:







       '''simple docstring'''



       warnings.warn(
           '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
           '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
           '''your audio inputs, or in a separate call.'''	)
       A:		List[Any]          =		True
       A:		List[str]          =		self.tokenizer
       yield
       A:		List[Any]          =		self.feature_extractor
       A:		int          =		False






