



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class       _lowerCAmelCase (			_UpperCAmelCase     ,       unittest.TestCase      ):






			"""simple docstring"""





			lowercase__   :	str  			=      RealmTokenizer
			lowercase__   :	Optional[Any]  			=      RealmTokenizerFast
			lowercase__   :	Optional[int]  			=      True
			lowercase__   :	int  			=      True
			lowercase__   :	Any  			=      filter_non_english





			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Dict:
									"""simple docstring"""







									super().setUp()

									__lowercase											=					[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									__lowercase											=					os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES["""vocab_file"""] )
									with open(self.vocab_file      ,  """w"""      ,  encoding="""utf-8""" ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )





			def 		snake_case__      (					self						:							Union[str, Any]      ,  lowercase						:							str )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					"""UNwant\u00E9d,running"""
									__lowercase											=					"""unwanted, running"""
									return input_text, output_text





			def 		snake_case__      (					self						:							List[Any] )    ->       Union[str, Any]:
									"""simple docstring"""







									__lowercase											=					self.tokenizer_class(self.vocab_file )

									__lowercase											=					tokenizer.tokenize("""UNwant\u00E9d,running""" )
									self.assertListEqual(lowercase      ,  ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""] )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase )      ,  [9, 6, 7, 12, 10, 11] )





			def 		snake_case__      (					self						:							List[Any] )    ->       Optional[int]:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
															return

									__lowercase											=					self.get_tokenizer()
									__lowercase											=					self.get_rust_tokenizer()

									__lowercase											=					"""UNwant\u00E9d,running"""

									__lowercase											=					tokenizer.tokenize(lowercase )
									__lowercase											=					rust_tokenizer.tokenize(lowercase )
									self.assertListEqual(lowercase      ,  lowercase )

									__lowercase											=					tokenizer.encode(lowercase      ,  add_special_tokens=lowercase )
									__lowercase											=					rust_tokenizer.encode(lowercase      ,  add_special_tokens=lowercase )
									self.assertListEqual(lowercase      ,  lowercase )

									__lowercase											=					self.get_rust_tokenizer()
									__lowercase											=					tokenizer.encode(lowercase )
									__lowercase											=					rust_tokenizer.encode(lowercase )
									self.assertListEqual(lowercase      ,  lowercase )

									# With lower casing
									__lowercase											=					self.get_tokenizer(do_lower_case=lowercase )
									__lowercase											=					self.get_rust_tokenizer(do_lower_case=lowercase )

									__lowercase											=					"""UNwant\u00E9d,running"""

									__lowercase											=					tokenizer.tokenize(lowercase )
									__lowercase											=					rust_tokenizer.tokenize(lowercase )
									self.assertListEqual(lowercase      ,  lowercase )

									__lowercase											=					tokenizer.encode(lowercase      ,  add_special_tokens=lowercase )
									__lowercase											=					rust_tokenizer.encode(lowercase      ,  add_special_tokens=lowercase )
									self.assertListEqual(lowercase      ,  lowercase )

									__lowercase											=					self.get_rust_tokenizer()
									__lowercase											=					tokenizer.encode(lowercase )
									__lowercase											=					rust_tokenizer.encode(lowercase )
									self.assertListEqual(lowercase      ,  lowercase )





			def 		snake_case__      (					self						:							int )    ->       Optional[Any]:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""" )      ,  ["""ah""", """\u535A""", """\u63A8""", """zz"""] )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       Tuple:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )      ,  ["""hello""", """!""", """how""", """are""", """you""", """?"""] )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )      ,  ["""hello"""] )





			def 		snake_case__      (					self						:							str )    ->       Any:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase      ,  strip_accents=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )      ,  ["""hällo""", """!""", """how""", """are""", """you""", """?"""] )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )      ,  ["""h\u00E9llo"""] )





			def 		snake_case__      (					self						:							Dict )    ->       str:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase      ,  strip_accents=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )      ,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )      ,  ["""hello"""] )





			def 		snake_case__      (					self						:							Tuple )    ->       int:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )      ,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""] )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""" )      ,  ["""hello"""] )





			def 		snake_case__      (					self						:							Dict )    ->       Union[str, Any]:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """ )      ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





			def 		snake_case__      (					self						:							Tuple )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase      ,  strip_accents=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )      ,  ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase      ,  strip_accents=lowercase )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """ )      ,  ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""] )





			def 		snake_case__      (					self						:							Union[str, Any] )    ->       Any:
									"""simple docstring"""







									__lowercase											=					BasicTokenizer(do_lower_case=lowercase      ,  never_split=["""[UNK]"""] )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""" )      ,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""] )





			def 		snake_case__      (					self						:							Optional[int] )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									__lowercase											=					{}
									for i, token in enumerate(lowercase ):
															__lowercase											=					i
									__lowercase											=					WordpieceTokenizer(vocab=lowercase      ,  unk_token="""[UNK]""" )

									self.assertListEqual(tokenizer.tokenize("""""" )      ,  [] )

									self.assertListEqual(tokenizer.tokenize("""unwanted running""" )      ,  ["""un""", """##want""", """##ed""", """runn""", """##ing"""] )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running""" )      ,  ["""[UNK]""", """runn""", """##ing"""] )





			def 		snake_case__      (					self						:							List[str] )    ->       Optional[int]:
									"""simple docstring"""







									self.assertTrue(_is_whitespace(""" """ ) )
									self.assertTrue(_is_whitespace("""\t""" ) )
									self.assertTrue(_is_whitespace("""\r""" ) )
									self.assertTrue(_is_whitespace("""\n""" ) )
									self.assertTrue(_is_whitespace("""\u00A0""" ) )

									self.assertFalse(_is_whitespace("""A""" ) )
									self.assertFalse(_is_whitespace("""-""" ) )





			def 		snake_case__      (					self						:							Optional[Any] )    ->       Optional[int]:
									"""simple docstring"""







									self.assertTrue(_is_control("""\u0005""" ) )

									self.assertFalse(_is_control("""A""" ) )
									self.assertFalse(_is_control(""" """ ) )
									self.assertFalse(_is_control("""\t""" ) )
									self.assertFalse(_is_control("""\r""" ) )





			def 		snake_case__      (					self						:							int )    ->       str:
									"""simple docstring"""







									self.assertTrue(_is_punctuation("""-""" ) )
									self.assertTrue(_is_punctuation("""$""" ) )
									self.assertTrue(_is_punctuation("""`""" ) )
									self.assertTrue(_is_punctuation(""".""" ) )

									self.assertFalse(_is_punctuation("""A""" ) )
									self.assertFalse(_is_punctuation(""" """ ) )





			def 		snake_case__      (					self						:							List[Any] )    ->       Optional[int]:
									"""simple docstring"""







									__lowercase											=					self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(lowercase ) for t in ["""Test""", """\xad""", """test"""]]      ,  [["""[UNK]"""], [], ["""[UNK]"""]] )

									if self.test_rust_tokenizer:
															__lowercase											=					self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(lowercase ) for t in ["""Test""", """\xad""", """test"""]]      ,  [["""[UNK]"""], [], ["""[UNK]"""]] )





			@slow
			def 		snake_case__      (					self						:							int )    ->       List[str]:
									"""simple docstring"""







									__lowercase											=					self.tokenizer_class.from_pretrained("""bert-base-uncased""" )

									__lowercase											=					tokenizer.encode("""sequence builders"""      ,  add_special_tokens=lowercase )
									__lowercase											=					tokenizer.encode("""multi-sequence build"""      ,  add_special_tokens=lowercase )

									__lowercase											=					tokenizer.build_inputs_with_special_tokens(lowercase )
									__lowercase											=					tokenizer.build_inputs_with_special_tokens(lowercase      ,  lowercase )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]





			def 		snake_case__      (					self						:							Optional[int] )    ->       Optional[Any]:
									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																					__lowercase											=					self.rust_tokenizer_class.from_pretrained(lowercase      ,  **lowercase )

																					__lowercase											=					F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
																					__lowercase											=					tokenizer_r.encode_plus(
																					    lowercase      ,  return_attention_mask=lowercase      ,  return_token_type_ids=lowercase      ,  return_offsets_mapping=lowercase      ,  add_special_tokens=lowercase      ,  )

																					__lowercase											=					tokenizer_r.do_lower_case if hasattr(lowercase      ,  """do_lower_case""" ) else False
																					__lowercase											=					(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """A"""),
																					        ((1, 2), ""","""),
																					        ((3, 5), """na"""),
																					        ((5, 6), """##ï"""),
																					        ((6, 8), """##ve"""),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), """Allen"""),
																					        ((21, 23), """##NL"""),
																					        ((23, 24), """##P"""),
																					        ((25, 33), """sentence"""),
																					        ((33, 34), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """a"""),
																					        ((1, 2), ""","""),
																					        ((3, 8), """naive"""),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), """allen"""),
																					        ((21, 23), """##nl"""),
																					        ((23, 24), """##p"""),
																					        ((25, 33), """sentence"""),
																					        ((33, 34), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""] ) )
																					self.assertEqual([e[0] for e in expected_results]      ,  tokens["""offset_mapping"""] )







			@slow
			def 		snake_case__      (					self						:							Dict )    ->       Union[str, Any]:
									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})" ):
																					__lowercase											=					self.rust_tokenizer_class.from_pretrained(lowercase      ,  **lowercase )
																					__lowercase											=					self.tokenizer_class.from_pretrained(lowercase      ,  **lowercase )
																					__lowercase											=					[["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

																					__lowercase											=					tokenizer_r.batch_encode_candidates(lowercase      ,  max_length=10      ,  return_tensors="""np""" )
																					__lowercase											=					tokenizer_p.batch_encode_candidates(lowercase      ,  max_length=10      ,  return_tensors="""np""" )

																					__lowercase											=					(2, 2, 10)

																					self.assertEqual(encoded_sentence_r["""input_ids"""].shape      ,  lowercase )
																					self.assertEqual(encoded_sentence_r["""attention_mask"""].shape      ,  lowercase )
																					self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape      ,  lowercase )

																					self.assertEqual(encoded_sentence_p["""input_ids"""].shape      ,  lowercase )
																					self.assertEqual(encoded_sentence_p["""attention_mask"""].shape      ,  lowercase )
																					self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape      ,  lowercase )










# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ..utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase__			=    {
    "agents": ["Agent", "AzureOpenAiAgent", "HfAgent", "LocalAgent", "OpenAiAgent"],
    "base": ["PipelineTool", "RemoteTool", "Tool", "launch_gradio_demo", "load_tool"],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCamelCase__			=    ["DocumentQuestionAnsweringTool"]
			UpperCamelCase__			=    ["ImageCaptioningTool"]
			UpperCamelCase__			=    ["ImageQuestionAnsweringTool"]
			UpperCamelCase__			=    ["ImageSegmentationTool"]
			UpperCamelCase__			=    ["SpeechToTextTool"]
			UpperCamelCase__			=    ["TextClassificationTool"]
			UpperCamelCase__			=    ["TextQuestionAnsweringTool"]
			UpperCamelCase__			=    ["TextSummarizationTool"]
			UpperCamelCase__			=    ["TextToSpeechTool"]
			UpperCamelCase__			=    ["TranslationTool"]

if TYPE_CHECKING:
			from .agents import Agent, AzureOpenAiAgent, HfAgent, LocalAgent, OpenAiAgent
			from .base import PipelineTool, RemoteTool, Tool, launch_gradio_demo, load_tool

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .document_question_answering import DocumentQuestionAnsweringTool
						from .image_captioning import ImageCaptioningTool
						from .image_question_answering import ImageQuestionAnsweringTool
						from .image_segmentation import ImageSegmentationTool
						from .speech_to_text import SpeechToTextTool
						from .text_classification import TextClassificationTool
						from .text_question_answering import TextQuestionAnsweringTool
						from .text_summarization import TextSummarizationTool
						from .text_to_speech import TextToSpeechTool
						from .translation import TranslationTool
else:
			import sys

			UpperCamelCase__			=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





