





import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
     from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
     from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
     from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__A									=				DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__A									=				"main"
# Default branch name
__A									=				"f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
__A									=				"aaaaaaa"
# This commit does not exist, so we should 404.

__A									=				"d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
__A									=				"4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"





@contextlib.contextmanager
def      lowerCAmelCase_ (       )     ->		List[Any]:







  """simple docstring"""





  print("Welcome!"		)
  yield
  print("Bye!"		)





@contextlib.contextmanager
def      lowerCAmelCase_ (       )     ->		int:







  """simple docstring"""





  print("Bonjour!"		)
  yield
  print("Au revoir!"		)



class    _SCREAMING_SNAKE_CASE						(	unittest.TestCase    ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_					(self				:   int)       ->Optional[int]:



   '''simple docstring'''



   assert transformers.__spec__ is not None
   assert importlib.util.find_spec("transformers") is not None



class    _SCREAMING_SNAKE_CASE						(	unittest.TestCase    ):







 '''simple docstring'''

 @unittest.mock.patch("sys.stdout"       , new_callable=io.StringIO)
 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any]       , UpperCAmelCase_				:   List[Any])       ->Dict:



   '''simple docstring'''



   with ContextManagers([]):
     print("Transformers are awesome!")
   # The print statement adds a new line at the end of the output
   self.assertEqual(mock_stdout.getvalue()       , "Transformers are awesome!\n")

 @unittest.mock.patch("sys.stdout"       , new_callable=io.StringIO)
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[str]       , UpperCAmelCase_				:   List[str])       ->Any:



   '''simple docstring'''



   with ContextManagers([context_en()]):
     print("Transformers are awesome!")
   # The output should be wrapped with an English welcome and goodbye
   self.assertEqual(mock_stdout.getvalue()       , "Welcome!\nTransformers are awesome!\nBye!\n")

 @unittest.mock.patch("sys.stdout"       , new_callable=io.StringIO)
 def 							SCREAMING_SNAKE_CASE_					(self				:   List[str]       , UpperCAmelCase_				:   Dict)       ->List[Any]:



   '''simple docstring'''



   with ContextManagers([context_fr(), context_en()]):
     print("Transformers are awesome!")
   # The output should be wrapped with an English and French welcome and goodbye
   self.assertEqual(mock_stdout.getvalue()       , "Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n")

 @require_torch
 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[Any])       ->Union[str, Any]:



   '''simple docstring'''



   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels"])
   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels", "next_sentence_label"])
   self.assertEqual(find_labels(UpperCAmelCase_)       , ["start_positions", "end_positions"])



   class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







    '''simple docstring'''
    pass



   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels"])

 @require_tf
 def 							SCREAMING_SNAKE_CASE_					(self				:   Dict)       ->Optional[Any]:



   '''simple docstring'''



   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels"])
   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels", "next_sentence_label"])
   self.assertEqual(find_labels(UpperCAmelCase_)       , ["start_positions", "end_positions"])



   class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







    '''simple docstring'''
    pass



   self.assertEqual(find_labels(UpperCAmelCase_)       , ["labels"])







 @require_flax
 def 							SCREAMING_SNAKE_CASE_					(self				:   Tuple)       ->Any:



   '''simple docstring'''



   self.assertEqual(find_labels(UpperCAmelCase_)       , [])
   self.assertEqual(find_labels(UpperCAmelCase_)       , [])
   self.assertEqual(find_labels(UpperCAmelCase_)       , [])



   class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ):







    '''simple docstring'''
    pass



   self.assertEqual(find_labels(UpperCAmelCase_)       , [])








import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


__A									=				get_tests_dir("fixtures/test_sentencepiece_bpe.model")



class    _SCREAMING_SNAKE_CASE						(	__SCREAMING_SNAKE_CASE    ,							unittest.TestCase    ):







 '''simple docstring'''
 lowercase_		=			BartphoTokenizer
 lowercase_		=			False
 lowercase_		=			True

 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Tuple:



   '''simple docstring'''



   super().setUp()

   lowerCamelCase__:      int			  =["▁This", "▁is", "▁a", "▁t", "est"]
   lowerCamelCase__:      Tuple			  =dict(zip(UpperCAmelCase_       , range(len(UpperCAmelCase_))))
   lowerCamelCase__:      List[Any]			  ={"unk_token": "<unk>"}

   lowerCamelCase__:      Dict			  =os.path.join(self.tmpdirname       , VOCAB_FILES_NAMES["monolingual_vocab_file"])
   with open(self.monolingual_vocab_file       , "w"       , encoding="utf-8") as fp:
     for token in vocab_tokens:
       fp.write(F"""{token} {vocab_tokens[token]}\n""")

   lowerCamelCase__:      Dict			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   tokenizer.save_pretrained(self.tmpdirname)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Optional[int]       , **UpperCAmelCase_				:   Optional[Any])       ->str:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return BartphoTokenizer.from_pretrained(self.tmpdirname       , **UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_					(self				:   Union[str, Any]       , UpperCAmelCase_				:   Optional[Any])       ->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:      Optional[int]			  ="This is a là test"
   lowerCamelCase__:      Optional[Any]			  ="This is a<unk><unk> test"
   return input_text, output_text







 def 							SCREAMING_SNAKE_CASE_					(self				:   List[Any])       ->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:      str			  =BartphoTokenizer(UpperCAmelCase_       , self.monolingual_vocab_file       , **self.special_tokens_map)
   lowerCamelCase__:      List[Any]			  ="This is a là test"
   lowerCamelCase__:      Optional[int]			  ="▁This ▁is ▁a ▁l à ▁t est".split()
   lowerCamelCase__:      Optional[int]			  =tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_       , UpperCAmelCase_)

   lowerCamelCase__:      Tuple			  =tokens + [tokenizer.unk_token]
   lowerCamelCase__:      List[Any]			  =[4, 5, 6, 3, 3, 7, 8, 3]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)       , UpperCAmelCase_)

