
import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 							UpperCAmelCase      (  __SCREAMING_SNAKE_CASE			,			unittest.TestCase  ):




	'''simple docstring'''





	snake_case_					    =     MobileBertTokenizer
	snake_case_					    =     MobileBertTokenizerFast
	snake_case_					    =     True
	snake_case_					    =     True
	snake_case_					    =     filter_non_english
	snake_case_					    =     "google/mobilebert-uncased"






	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					super().setUp()

					__A   			= [
					    "[UNK]",
					    "[CLS]",
					    "[SEP]",
					    "[PAD]",
					    "[MASK]",
					    "want",
					    "##want",
					    "##ed",
					    "wa",
					    "un",
					    "runn",
					    "##ing",
					    ",",
					    "low",
					    "lowest",
					]
					__A   			= os.path.join(self.tmpdirname							,VOCAB_FILES_NAMES["vocab_file"] )
					with open(self.vocab_file							,"w"							,encoding="utf-8" ) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )

					__A   			= [
					    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
					    for tokenizer_def in self.tokenizers_list
					]






	def 				UpperCamelCase_  (       self					: Tuple							,A					: List[str] ):
					__A   			= "UNwant\u00E9d,running"
					__A   			= "unwanted, running"
					return input_text, output_text






	def 				UpperCamelCase_  (       self					: str ):
					__A   			= self.tokenizer_class(self.vocab_file )

					__A   			= tokenizer.tokenize("UNwant\u00E9d,running" )
					self.assertListEqual(A							,["un", "##want", "##ed", ",", "runn", "##ing"] )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(A )							,[9, 6, 7, 12, 10, 11] )






	def 				UpperCamelCase_  (       self					: List[Any] ):
					if not self.test_rust_tokenizer:
									return

					__A   			= self.get_tokenizer()
					__A   			= self.get_rust_tokenizer()

					__A   			= "UNwant\u00E9d,running"

					__A   			= tokenizer.tokenize(A )
					__A   			= rust_tokenizer.tokenize(A )
					self.assertListEqual(A							,A )

					__A   			= tokenizer.encode(A							,add_special_tokens=A )
					__A   			= rust_tokenizer.encode(A							,add_special_tokens=A )
					self.assertListEqual(A							,A )

					__A   			= self.get_rust_tokenizer()
					__A   			= tokenizer.encode(A )
					__A   			= rust_tokenizer.encode(A )
					self.assertListEqual(A							,A )

					# With lower casing
					__A   			= self.get_tokenizer(do_lower_case=A )
					__A   			= self.get_rust_tokenizer(do_lower_case=A )

					__A   			= "UNwant\u00E9d,running"

					__A   			= tokenizer.tokenize(A )
					__A   			= rust_tokenizer.tokenize(A )
					self.assertListEqual(A							,A )

					__A   			= tokenizer.encode(A							,add_special_tokens=A )
					__A   			= rust_tokenizer.encode(A							,add_special_tokens=A )
					self.assertListEqual(A							,A )

					__A   			= self.get_rust_tokenizer()
					__A   			= tokenizer.encode(A )
					__A   			= rust_tokenizer.encode(A )
					self.assertListEqual(A							,A )






	def 				UpperCamelCase_  (       self					: List[str] ):
					__A   			= BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz" )							,["ah", "\u535A", "\u63A8", "zz"] )






	def 				UpperCamelCase_  (       self					: int ):
					__A   			= BasicTokenizer(do_lower_case=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )							,["hello", "!", "how", "are", "you", "?"] )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )							,["hello"] )






	def 				UpperCamelCase_  (       self					: Tuple ):
					__A   			= BasicTokenizer(do_lower_case=A							,strip_accents=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )							,["hällo", "!", "how", "are", "you", "?"] )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )							,["h\u00E9llo"] )






	def 				UpperCamelCase_  (       self					: List[Any] ):
					__A   			= BasicTokenizer(do_lower_case=A							,strip_accents=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )							,["hallo", "!", "how", "are", "you", "?"] )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )							,["hello"] )






	def 				UpperCamelCase_  (       self					: str ):
					__A   			= BasicTokenizer(do_lower_case=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )							,["hallo", "!", "how", "are", "you", "?"] )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo" )							,["hello"] )






	def 				UpperCamelCase_  (       self					: List[str] ):
					__A   			= BasicTokenizer(do_lower_case=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  " )							,["HeLLo", "!", "how", "Are", "yoU", "?"] )






	def 				UpperCamelCase_  (       self					: Tuple ):
					__A   			= BasicTokenizer(do_lower_case=A							,strip_accents=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )							,["HäLLo", "!", "how", "Are", "yoU", "?"] )






	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					__A   			= BasicTokenizer(do_lower_case=A							,strip_accents=A )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  " )							,["HaLLo", "!", "how", "Are", "yoU", "?"] )






	def 				UpperCamelCase_  (       self					: str ):
					__A   			= BasicTokenizer(do_lower_case=A							,never_split=["[UNK]"] )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]" )							,["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] )






	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					__A   			= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

					__A   			= {}
					for i, token in enumerate(A ):
									__A   			= i
					__A   			= WordpieceTokenizer(vocab=A							,unk_token="[UNK]" )

					self.assertListEqual(tokenizer.tokenize("" )							,[] )

					self.assertListEqual(tokenizer.tokenize("unwanted running" )							,["un", "##want", "##ed", "runn", "##ing"] )

					self.assertListEqual(tokenizer.tokenize("unwantedX running" )							,["[UNK]", "runn", "##ing"] )






	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					self.assertTrue(_is_whitespace(" " ) )
					self.assertTrue(_is_whitespace("\t" ) )
					self.assertTrue(_is_whitespace("\r" ) )
					self.assertTrue(_is_whitespace("\n" ) )
					self.assertTrue(_is_whitespace("\u00A0" ) )

					self.assertFalse(_is_whitespace("A" ) )
					self.assertFalse(_is_whitespace("-" ) )






	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					self.assertTrue(_is_control("\u0005" ) )

					self.assertFalse(_is_control("A" ) )
					self.assertFalse(_is_control(" " ) )
					self.assertFalse(_is_control("\t" ) )
					self.assertFalse(_is_control("\r" ) )






	def 				UpperCamelCase_  (       self					: Optional[int] ):
					self.assertTrue(_is_punctuation("-" ) )
					self.assertTrue(_is_punctuation("$" ) )
					self.assertTrue(_is_punctuation("`" ) )
					self.assertTrue(_is_punctuation("." ) )

					self.assertFalse(_is_punctuation("A" ) )
					self.assertFalse(_is_punctuation(" " ) )






	def 				UpperCamelCase_  (       self					: List[Any] ):
					__A   			= self.get_tokenizer()
					__A   			= self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(A ) for t in ["Test", "\xad", "test"]]							,[["[UNK]"], [], ["[UNK]"]] )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(A ) for t in ["Test", "\xad", "test"]]							,[["[UNK]"], [], ["[UNK]"]] )






	@slow
	def 				UpperCamelCase_  (       self					: Any ):
					__A   			= self.tokenizer_class.from_pretrained("google/mobilebert-uncased" )

					__A   			= tokenizer.encode("sequence builders"							,add_special_tokens=A )
					__A   			= tokenizer.encode("multi-sequence build"							,add_special_tokens=A )

					__A   			= tokenizer.build_inputs_with_special_tokens(A )
					__A   			= tokenizer.build_inputs_with_special_tokens(A							,A )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]






	def 				UpperCamelCase_  (       self					: str ):
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
													__A   			= self.rust_tokenizer_class.from_pretrained(A							,**A )

													__A   			= f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													__A   			= tokenizer_r.encode_plus(
													    A							,return_attention_mask=A							,return_token_type_ids=A							,return_offsets_mapping=A							,add_special_tokens=A							,)

													__A   			= tokenizer_r.do_lower_case if hasattr(A							,"do_lower_case" ) else False
													__A   			= (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]							,tokenizer_r.convert_ids_to_tokens(tokens["input_ids"] ) )
													self.assertEqual([e[0] for e in expected_results]							,tokens["offset_mapping"] )







	def 				UpperCamelCase_  (       self					: Optional[int] ):
					__A   			= ["的", "人", "有"]
					__A   			= "".join(A )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
													__A   			= True
													__A   			= self.tokenizer_class.from_pretrained(A							,**A )
													__A   			= self.rust_tokenizer_class.from_pretrained(A							,**A )

													__A   			= tokenizer_p.encode(A							,add_special_tokens=A )
													__A   			= tokenizer_r.encode(A							,add_special_tokens=A )

													__A   			= tokenizer_r.convert_ids_to_tokens(A )
													__A   			= tokenizer_p.convert_ids_to_tokens(A )

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(A							,A )
													self.assertListEqual(A							,A )

													__A   			= False
													__A   			= self.rust_tokenizer_class.from_pretrained(A							,**A )
													__A   			= self.tokenizer_class.from_pretrained(A							,**A )

													__A   			= tokenizer_r.encode(A							,add_special_tokens=A )
													__A   			= tokenizer_p.encode(A							,add_special_tokens=A )

													__A   			= tokenizer_r.convert_ids_to_tokens(A )
													__A   			= tokenizer_p.convert_ids_to_tokens(A )

													# it is expected that only the first Chinese character is not preceded by "##".
													__A   			= [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(A )
													]
													self.assertListEqual(A							,A )
													self.assertListEqual(A							,A )



"""simple docstring"""





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool




class 					_lowerCAmelCase  (	lowercase   ):







 """simple docstring"""







 __UpperCAmelCase	:					Tuple   		=	"openai/whisper-base"
 __UpperCAmelCase	:					Union[str, Any]   		=	(
     "This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the "
     "transcribed text."
 )
 __UpperCAmelCase	:					List[str]   		=	"transcriber"
 __UpperCAmelCase	:					Optional[Any]   		=	WhisperProcessor
 __UpperCAmelCase	:					str   		=	WhisperForConditionalGeneration

 __UpperCAmelCase	:					List[str]   		=	["audio"]
 __UpperCAmelCase	:					Tuple   		=	["text"]


 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			int	):
    return self.pre_processor(UpperCAmelCase__,					return_tensors="pt"	).input_features


 def 					_lowercase (			self							:			Union[str, Any],					UpperCAmelCase__							:			Optional[Any]	):
    return self.model.generate(inputs=UpperCAmelCase__	)



 def 					_lowercase (			self							:			Dict,					UpperCAmelCase__							:			Optional[int]	):
    return self.pre_processor.batch_decode(UpperCAmelCase__,					skip_special_tokens=UpperCAmelCase__	)[0]




