
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class   UpperCAmelCase		(						A_       ,unittest.TestCase ):
   A__					:		int						     =     BertJapaneseTokenizer
   A__					:		Dict						     =     False
   A__					:		str						     =     True



   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	str:


          '''simple docstring'''


          super().setUp()

          snake_case       :   Optional[Any]							=      [
              '''[UNK]''',
              '''[CLS]''',
              '''[SEP]''',
              '''こんにちは''',
              '''こん''',
              '''にちは''',
              '''ばんは''',
              '''##こん''',
              '''##にちは''',
              '''##ばんは''',
              '''世界''',
              '''##世界''',
              '''、''',
              '''##、''',
              '''。''',
              '''##。''',
          ]

          snake_case       :   Dict							=      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"] )
          with open(self.vocab_file    ,						"w"    ,						encoding="utf-8" ) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						snake_case__		:					Dict )		->	str:


          '''simple docstring'''


          snake_case       :   str							=      '''こんにちは、世界。 \nこんばんは、世界。'''
          snake_case       :   Any							=      '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
          return input_text, output_text



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int]    ,						snake_case__		:					str )		->	Any:


          '''simple docstring'''


          snake_case       :   List[Any]							=      self.get_input_output_texts(_snake_case )
          snake_case       :   int							=      tokenizer.encode(_snake_case    ,						add_special_tokens=_snake_case )
          snake_case       :   Optional[Any]							=      tokenizer.decode(_snake_case    ,						clean_up_tokenization_spaces=_snake_case )
          return text, ids



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any] )		->	int:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	str:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					Tuple )		->	List[str]:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	List[Any]:


          '''simple docstring'''


          snake_case       :   Tuple							=      self.tokenizer_class(self.vocab_file )

          snake_case       :   Any							=      tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。" )
          self.assertListEqual(_snake_case    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	Any:


          '''simple docstring'''


          snake_case       :   List[str]							=      self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="mecab" )
          self.assertIsNotNone(_snake_case )

          snake_case       :   Optional[Any]							=      '''こんにちは、世界。\nこんばんは、世界。'''
          snake_case       :   Optional[int]							=      tokenizer.tokenize(_snake_case )
          self.assertListEqual(_snake_case    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

          snake_case       :   Dict							=      os.path.join(self.tmpdirname    ,						"tokenizer.bin" )
          with open(_snake_case    ,						"wb" ) as handle:
                 pickle.dump(_snake_case    ,						_snake_case )

          with open(_snake_case    ,						"rb" ) as handle:
                 snake_case       :   int							=      pickle.load(_snake_case )

          snake_case       :   Optional[Any]							=      tokenizer_new.tokenize(_snake_case )

          self.assertListEqual(_snake_case    ,						_snake_case )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	Union[str, Any]:


          '''simple docstring'''


          snake_case       :   Tuple							=      MecabTokenizer(mecab_dic="ipadic" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Tuple:


          '''simple docstring'''


          try:
                 snake_case       :   Optional[Any]							=      MecabTokenizer(mecab_dic="unidic_lite" )
          except ModuleNotFoundError:
                 return

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Any:


          '''simple docstring'''


          try:
                 snake_case       :   Dict							=      MecabTokenizer(mecab_dic="unidic" )
          except ModuleNotFoundError:
                 return

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   int							=      MecabTokenizer(do_lower_case=_snake_case    ,						mecab_dic="ipadic" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any] )		->	Union[str, Any]:


          '''simple docstring'''


          try:
                 snake_case       :   Tuple							=      MecabTokenizer(
                     do_lower_case=_snake_case    ,						normalize_text=_snake_case    ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic" )
          except RuntimeError:
                 # if dict doesn't exist in the system, previous code raises this error.
                 return

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str] )		->	int:


          '''simple docstring'''


          snake_case       :   str							=      MecabTokenizer(normalize_text=_snake_case    ,						mecab_dic="ipadic" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]    ,						)



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str] )		->	List[str]:


          '''simple docstring'''


          snake_case       :   Optional[Any]							=      self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="sudachi" )
          self.assertIsNotNone(_snake_case )

          snake_case       :   Tuple							=      '''こんにちは、世界。\nこんばんは、世界。'''
          snake_case       :   Optional[int]							=      tokenizer.tokenize(_snake_case )
          self.assertListEqual(_snake_case    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

          snake_case       :   Optional[Any]							=      os.path.join(self.tmpdirname    ,						"tokenizer.bin" )
          with open(_snake_case    ,						"wb" ) as handle:
                 pickle.dump(_snake_case    ,						_snake_case )

          with open(_snake_case    ,						"rb" ) as handle:
                 snake_case       :   Optional[int]							=      pickle.load(_snake_case )

          snake_case       :   Dict							=      tokenizer_new.tokenize(_snake_case )

          self.assertListEqual(_snake_case    ,						_snake_case )



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	str:


          '''simple docstring'''


          snake_case       :   int							=      SudachiTokenizer(sudachi_dict_type="core" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	Union[str, Any]:


          '''simple docstring'''


          snake_case       :   Optional[Any]							=      SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="A" )

          self.assertListEqual(tokenizer.tokenize("外国人参政権" )    ,						["外国", "人", "参政", "権"] )



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Union[str, Any]:


          '''simple docstring'''


          snake_case       :   int							=      SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="B" )

          self.assertListEqual(tokenizer.tokenize("外国人参政権" )    ,						["外国人", "参政権"] )



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	int:


          '''simple docstring'''


          snake_case       :   Optional[int]							=      SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="C" )

          self.assertListEqual(tokenizer.tokenize("外国人参政権" )    ,						["外国人参政権"] )



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any] )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   Dict							=      SudachiTokenizer(do_lower_case=_snake_case    ,						sudachi_dict_type="core" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	int:


          '''simple docstring'''


          snake_case       :   Optional[Any]							=      SudachiTokenizer(normalize_text=_snake_case    ,						sudachi_dict_type="core" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]    ,						)



   @require_sudachi
   def 				_SCREAMING_SNAKE_CASE   (self		:					Tuple )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   Optional[Any]							=      SudachiTokenizer(trim_whitespace=_snake_case    ,						sudachi_dict_type="core" )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any] )		->	str:


          '''simple docstring'''


          snake_case       :   str							=      self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="jumanpp" )
          self.assertIsNotNone(_snake_case )

          snake_case       :   Dict							=      '''こんにちは、世界。\nこんばんは、世界。'''
          snake_case       :   List[Any]							=      tokenizer.tokenize(_snake_case )
          self.assertListEqual(_snake_case    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case )    ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

          snake_case       :   int							=      os.path.join(self.tmpdirname    ,						"tokenizer.bin" )
          with open(_snake_case    ,						"wb" ) as handle:
                 pickle.dump(_snake_case    ,						_snake_case )

          with open(_snake_case    ,						"rb" ) as handle:
                 snake_case       :   Optional[int]							=      pickle.load(_snake_case )

          snake_case       :   Tuple							=      tokenizer_new.tokenize(_snake_case )

          self.assertListEqual(_snake_case    ,						_snake_case )



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	Tuple:


          '''simple docstring'''


          snake_case       :   List[str]							=      JumanppTokenizer()

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	Any:


          '''simple docstring'''


          snake_case       :   List[str]							=      JumanppTokenizer(do_lower_case=_snake_case )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Tuple )		->	Any:


          '''simple docstring'''


          snake_case       :   Any							=      JumanppTokenizer(normalize_text=_snake_case )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	str:


          '''simple docstring'''


          snake_case       :   int							=      JumanppTokenizer(trim_whitespace=_snake_case )

          self.assertListEqual(
              tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]    ,						)



   @require_jumanpp
   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   int							=      JumanppTokenizer()

          self.assertListEqual(
              tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。" )    ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]    ,						)



   def 				_SCREAMING_SNAKE_CASE   (self		:					int )		->	List[str]:


          '''simple docstring'''


          snake_case       :   Union[str, Any]							=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

          snake_case       :   Dict							=      {}
          for i, token in enumerate(_snake_case ):
                 snake_case       :   Any							=      i
          snake_case       :   Any							=      WordpieceTokenizer(vocab=_snake_case    ,						unk_token="[UNK]" )

          self.assertListEqual(tokenizer.tokenize("" )    ,						[] )

          self.assertListEqual(tokenizer.tokenize("こんにちは" )    ,						["こんにちは"] )

          self.assertListEqual(tokenizer.tokenize("こんばんは" )    ,						["こん", "##ばんは"] )

          self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは" )    ,						["こん", "##ばんは", "[UNK]", "こんにちは"] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str] )		->	List[Any]:


          '''simple docstring'''


          snake_case       :   Dict							=      BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp" )
          snake_case       :   Union[str, Any]							=      tokenizer.subword_tokenizer

          snake_case       :   Tuple							=      subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。" )
          self.assertListEqual(_snake_case    ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"] )

          snake_case       :   List[Any]							=      subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは" )
          self.assertListEqual(_snake_case    ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any] )		->	Any:


          '''simple docstring'''


          snake_case       :   Any							=      self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese" )

          snake_case       :   List[Any]							=      tokenizer.encode("ありがとう。"    ,						add_special_tokens=_snake_case )
          snake_case       :   Optional[Any]							=      tokenizer.encode("どういたしまして。"    ,						add_special_tokens=_snake_case )

          snake_case       :   List[Any]							=      tokenizer.build_inputs_with_special_tokens(_snake_case )
          snake_case       :   str							=      tokenizer.build_inputs_with_special_tokens(_snake_case    ,						_snake_case )

          # 2 is for "[CLS]", 3 is for "[SEP]"
          assert encoded_sentence == [2] + text + [3]
          assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class   UpperCAmelCase		(						A_       ,unittest.TestCase ):
   A__					:		Optional[Any]						     =     BertJapaneseTokenizer
   A__					:		Tuple						     =     False



   def 				_SCREAMING_SNAKE_CASE   (self		:					str )		->	Tuple:


          '''simple docstring'''


          super().setUp()

          snake_case       :   List[Any]							=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

          snake_case       :   List[Any]							=      os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"] )
          with open(self.vocab_file    ,						"w"    ,						encoding="utf-8" ) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[Any]    ,						**snake_case__		:					int )		->	Optional[Any]:


          '''simple docstring'''


          return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="character"    ,						**_snake_case )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any    ,						snake_case__		:					List[Any] )		->	List[str]:


          '''simple docstring'''


          snake_case       :   Union[str, Any]							=      '''こんにちは、世界。 \nこんばんは、世界。'''
          snake_case       :   Optional[int]							=      '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
          return input_text, output_text



   def 				_SCREAMING_SNAKE_CASE   (self		:					Union[str, Any] )		->	int:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Dict:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[Any] )		->	Optional[int]:


          '''simple docstring'''


          pass  # TODO add if relevant



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	str:


          '''simple docstring'''


          snake_case       :   int							=      self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="character" )

          snake_case       :   Union[str, Any]							=      tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。" )
          self.assertListEqual(
              _snake_case    ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"] )
          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_snake_case )    ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Optional[int] )		->	Optional[int]:


          '''simple docstring'''


          snake_case       :   Tuple							=      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

          snake_case       :   Optional[Any]							=      {}
          for i, token in enumerate(_snake_case ):
                 snake_case       :   List[str]							=      i
          snake_case       :   Optional[Any]							=      CharacterTokenizer(vocab=_snake_case    ,						unk_token="[UNK]" )

          self.assertListEqual(tokenizer.tokenize("" )    ,						[] )

          self.assertListEqual(tokenizer.tokenize("こんにちは" )    ,						["こ", "ん", "に", "ち", "は"] )

          self.assertListEqual(tokenizer.tokenize("こんにちほ" )    ,						["こ", "ん", "に", "ち", "[UNK]"] )



   def 				_SCREAMING_SNAKE_CASE   (self		:					Dict )		->	Any:


          '''simple docstring'''


          snake_case       :   str							=      self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char" )

          snake_case       :   Dict							=      tokenizer.encode("ありがとう。"    ,						add_special_tokens=_snake_case )
          snake_case       :   Union[str, Any]							=      tokenizer.encode("どういたしまして。"    ,						add_special_tokens=_snake_case )

          snake_case       :   str							=      tokenizer.build_inputs_with_special_tokens(_snake_case )
          snake_case       :   Optional[int]							=      tokenizer.build_inputs_with_special_tokens(_snake_case    ,						_snake_case )

          # 2 is for "[CLS]", 3 is for "[SEP]"
          assert encoded_sentence == [2] + text + [3]
          assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class   UpperCAmelCase		(						unittest.TestCase ):



   def 				_SCREAMING_SNAKE_CASE   (self		:					Any )		->	List[str]:


          '''simple docstring'''


          snake_case       :   Dict							=      '''cl-tohoku/bert-base-japanese'''
          snake_case       :   List[str]							=      AutoTokenizer.from_pretrained(_snake_case )
          self.assertIsInstance(_snake_case    ,						_snake_case )






class   UpperCAmelCase		(						unittest.TestCase ):



   def 				_SCREAMING_SNAKE_CASE   (self		:					List[str] )		->	int:


          '''simple docstring'''


          snake_case       :   List[str]							=      '''cl-tohoku/bert-base-japanese'''
          with self.assertLogs("transformers"    ,						level="WARNING" ) as cm:
                 BertTokenizer.from_pretrained(_snake_case )
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from." ) )
          snake_case       :   List[str]							=      '''bert-base-cased'''
          with self.assertLogs("transformers"    ,						level="WARNING" ) as cm:
                 BertJapaneseTokenizer.from_pretrained(_snake_case )
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from." ) )










"""simple docstring"""







from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
							from transformers import (
							    AutoConfig,
							    BertConfig,
							    GPTaConfig,
							    TaConfig,
							    TFAutoModel,
							    TFAutoModelForCausalLM,
							    TFAutoModelForMaskedLM,
							    TFAutoModelForPreTraining,
							    TFAutoModelForQuestionAnswering,
							    TFAutoModelForSeqaSeqLM,
							    TFAutoModelForSequenceClassification,
							    TFAutoModelWithLMHead,
							    TFBertForMaskedLM,
							    TFBertForPreTraining,
							    TFBertForQuestionAnswering,
							    TFBertForSequenceClassification,
							    TFBertModel,
							    TFGPTaLMHeadModel,
							    TFRobertaForMaskedLM,
							    TFTaForConditionalGeneration,
							)
							from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
							from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
							from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
							from transformers import (
							    AutoModel,
							    AutoModelForCausalLM,
							    AutoModelForMaskedLM,
							    AutoModelForPreTraining,
							    AutoModelForQuestionAnswering,
							    AutoModelForSeqaSeqLM,
							    AutoModelForSequenceClassification,
							    AutoModelWithLMHead,
							    BertForMaskedLM,
							    BertForPreTraining,
							    BertForQuestionAnswering,
							    BertForSequenceClassification,
							    BertModel,
							    GPTaLMHeadModel,
							    RobertaForMaskedLM,
							    TaForConditionalGeneration,
							)
@is_pt_tf_cross_test
class        __A       (	unittest.TestCase    ):







		'''simple docstring'''





		@slow
		def 					UpperCAmelCase    (	self						:  Optional[Any]						)					->		Union[str, Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Tuple				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Dict				=			TFAutoModel.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			AutoModel.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Union[str, Any]						)					->		Tuple:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Dict				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    str				=			TFAutoModelForPreTraining.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Optional[Any]				=			AutoModelForPreTraining.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Tuple						)					->		Dict:






				"""simple docstring"""



				for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Any				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			TFAutoModelForCausalLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[Any]				=			TFAutoModelForCausalLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			AutoModelForCausalLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[Any]				=			AutoModelForCausalLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Any						)					->		Tuple:






				"""simple docstring"""



				for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Any				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Optional[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[str]						)					->		Optional[Any]:






				"""simple docstring"""



				for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    str				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			TFAutoModelForMaskedLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    str				=			TFAutoModelForMaskedLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			AutoModelForMaskedLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Any				=			AutoModelForMaskedLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[Any]						)					->		Dict:






				"""simple docstring"""



				for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__   :    Union[str, Any]				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    List[str]				=			TFAutoModelForSeqaSeqLM.from_pretrained(_snake_case			,from_pt=_snake_case						)
						lowercase__		,			lowercase__   :    List[str]				=			TFAutoModelForSeqaSeqLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelForSeqaSeqLM.from_pretrained(_snake_case			,from_tf=_snake_case						)
						lowercase__		,			lowercase__   :    Optional[int]				=			AutoModelForSeqaSeqLM.from_pretrained(
						    _snake_case			,output_loading_info=_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  List[str]						)					->		Union[str, Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    Tuple				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			TFAutoModelForSequenceClassification.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Union[str, Any]				=			AutoModelForSequenceClassification.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		@slow
		def 					UpperCAmelCase    (	self						:  Tuple						)					->		Optional[Any]:






				"""simple docstring"""



				for model_name in ["bert-base-uncased"]:
						lowercase__   :    List[Any]				=			AutoConfig.from_pretrained(_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    str				=			TFAutoModelForQuestionAnswering.from_pretrained(_snake_case			,from_pt=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)

						lowercase__   :    Any				=			AutoModelForQuestionAnswering.from_pretrained(_snake_case			,from_tf=_snake_case						)
						self.assertIsNotNone(_snake_case						)
						self.assertIsInstance(_snake_case			,_snake_case						)







		def 					UpperCAmelCase    (	self						:  Dict						)					->		Any:






				"""simple docstring"""



				lowercase__   :    Optional[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)

				lowercase__   :    Union[str, Any]				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)







		def 					UpperCAmelCase    (	self						:  int						)					->		List[Any]:






				"""simple docstring"""



				lowercase__   :    List[Any]				=			TFAutoModelWithLMHead.from_pretrained(_snake_case			,from_pt=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)

				lowercase__   :    int				=			AutoModelWithLMHead.from_pretrained(_snake_case			,from_tf=_snake_case						)
				self.assertIsInstance(_snake_case			,_snake_case						)
				self.assertEqual(model.num_parameters()			,14_410						)
				self.assertEqual(model.num_parameters(only_trainable=_snake_case						)			,14_410						)
