
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class __snake_case		(      SCREAMING_SNAKE_CASE_					,	unittest.TestCase     ):
    _a		:						List[str]=				BertJapaneseTokenizer
    _a		:						Optional[Any]=				False
    _a		:						List[Any]=				True
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           super().setUp()

           lowercase							:     List[Any]       	=      [
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """こんにちは""",
               """こん""",
               """にちは""",
               """ばんは""",
               """##こん""",
               """##にちは""",
               """##ばんは""",
               """世界""",
               """##世界""",
               """、""",
               """##、""",
               """。""",
               """##。""",
           ]

           lowercase							:     Union[str, Any]       	=      os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES["""vocab_file"""]      )
           with open(self.vocab_file		,"""w"""		,encoding="""utf-8"""      ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     int       	=      """こんにちは、世界。 \nこんばんは、世界。"""
           lowercase							:     Tuple       	=      """こんにちは 、 世界 。 こんばんは 、 世界 。"""
           return input_text, output_text
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.get_input_output_texts(a__      )
           lowercase							:     List[Any]       	=      tokenizer.encode(a__		,add_special_tokens=a__      )
           lowercase							:     Optional[int]       	=      tokenizer.decode(a__		,clean_up_tokenization_spaces=a__      )
           return text, ids
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.tokenizer_class(self.vocab_file      )

           lowercase							:     str       	=      tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""      )
           self.assertListEqual(a__		,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )		,[3, 12, 10, 14, 4, 9, 12, 10, 14]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      self.tokenizer_class(self.vocab_file		,word_tokenizer_type="""mecab"""      )
           self.assertIsNotNone(a__      )

           lowercase							:     str       	=      """こんにちは、世界。\nこんばんは、世界。"""
           lowercase							:     int       	=      tokenizer.tokenize(a__      )
           self.assertListEqual(a__		,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )		,[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

           lowercase							:     Dict       	=      os.path.join(self.tmpdirname		,"""tokenizer.bin"""      )
           with open(a__		,"""wb"""      ) as handle:
                  pickle.dump(a__		,a__      )

           with open(a__		,"""rb"""      ) as handle:
                  lowercase							:     List[Any]       	=      pickle.load(a__      )

           lowercase							:     List[Any]       	=      tokenizer_new.tokenize(a__      )

           self.assertListEqual(a__		,a__      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      MecabTokenizer(mecab_dic="""ipadic"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           try:
                  lowercase							:     List[Any]       	=      MecabTokenizer(mecab_dic="""unidic_lite"""      )
           except ModuleNotFoundError:
                  return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           try:
                  lowercase							:     str       	=      MecabTokenizer(mecab_dic="""unidic"""      )
           except ModuleNotFoundError:
                  return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      MecabTokenizer(do_lower_case=a__		,mecab_dic="""ipadic"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           try:
                  lowercase							:     Any       	=      MecabTokenizer(
                      do_lower_case=a__		,normalize_text=a__		,mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""      )
           except RuntimeError:
                  # if dict doesn't exist in the system, previous code raises this error.
                  return

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      MecabTokenizer(normalize_text=a__		,mecab_dic="""ipadic"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]		,)
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      self.tokenizer_class(self.vocab_file		,word_tokenizer_type="""sudachi"""      )
           self.assertIsNotNone(a__      )

           lowercase							:     List[str]       	=      """こんにちは、世界。\nこんばんは、世界。"""
           lowercase							:     int       	=      tokenizer.tokenize(a__      )
           self.assertListEqual(a__		,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )		,[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

           lowercase							:     Tuple       	=      os.path.join(self.tmpdirname		,"""tokenizer.bin"""      )
           with open(a__		,"""wb"""      ) as handle:
                  pickle.dump(a__		,a__      )

           with open(a__		,"""rb"""      ) as handle:
                  lowercase							:     Union[str, Any]       	=      pickle.load(a__      )

           lowercase							:     Tuple       	=      tokenizer_new.tokenize(a__      )

           self.assertListEqual(a__		,a__      )
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      SudachiTokenizer(sudachi_dict_type="""core"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]		,)
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      SudachiTokenizer(sudachi_dict_type="""core"""		,sudachi_split_mode="""A"""      )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,["""外国""", """人""", """参政""", """権"""]      )
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      SudachiTokenizer(sudachi_dict_type="""core"""		,sudachi_split_mode="""B"""      )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,["""外国人""", """参政権"""]      )
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      SudachiTokenizer(sudachi_dict_type="""core"""		,sudachi_split_mode="""C"""      )

           self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,["""外国人参政権"""]      )
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      SudachiTokenizer(do_lower_case=a__		,sudachi_dict_type="""core"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]		,)
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      SudachiTokenizer(normalize_text=a__		,sudachi_dict_type="""core"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]		,)
    @require_sudachi
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      SudachiTokenizer(trim_whitespace=a__		,sudachi_dict_type="""core"""      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,)
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      self.tokenizer_class(self.vocab_file		,word_tokenizer_type="""jumanpp"""      )
           self.assertIsNotNone(a__      )

           lowercase							:     Any       	=      """こんにちは、世界。\nこんばんは、世界。"""
           lowercase							:     Union[str, Any]       	=      tokenizer.tokenize(a__      )
           self.assertListEqual(a__		,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )		,[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

           lowercase							:     Optional[int]       	=      os.path.join(self.tmpdirname		,"""tokenizer.bin"""      )
           with open(a__		,"""wb"""      ) as handle:
                  pickle.dump(a__		,a__      )

           with open(a__		,"""rb"""      ) as handle:
                  lowercase							:     List[Any]       	=      pickle.load(a__      )

           lowercase							:     str       	=      tokenizer_new.tokenize(a__      )

           self.assertListEqual(a__		,a__      )
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,)
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      JumanppTokenizer(do_lower_case=a__      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,)
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      JumanppTokenizer(normalize_text=a__      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,)
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      JumanppTokenizer(trim_whitespace=a__      )

           self.assertListEqual(
               tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]		,)
    @require_jumanpp
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      JumanppTokenizer()

           self.assertListEqual(
               tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""      )		,["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]		,)
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

           lowercase							:     Union[str, Any]       	=      {}
           for i, token in enumerate(a__      ):
                  lowercase							:     List[str]       	=      i
           lowercase							:     Optional[int]       	=      WordpieceTokenizer(vocab=a__		,unk_token="""[UNK]"""      )

           self.assertListEqual(tokenizer.tokenize(""""""      )		,[]      )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""      )		,["""こんにちは"""]      )

           self.assertListEqual(tokenizer.tokenize("""こんばんは"""      )		,["""こん""", """##ばんは"""]      )

           self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""      )		,["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Any       	=      BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""      )
           lowercase							:     Union[str, Any]       	=      tokenizer.subword_tokenizer

           lowercase							:     Any       	=      subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""      )
           self.assertListEqual(a__		,["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]      )

           lowercase							:     Optional[int]       	=      subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""      )
           self.assertListEqual(a__		,["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""      )

           lowercase							:     Union[str, Any]       	=      tokenizer.encode("""ありがとう。"""		,add_special_tokens=a__      )
           lowercase							:     Optional[Any]       	=      tokenizer.encode("""どういたしまして。"""		,add_special_tokens=a__      )

           lowercase							:     List[str]       	=      tokenizer.build_inputs_with_special_tokens(a__      )
           lowercase							:     Any       	=      tokenizer.build_inputs_with_special_tokens(a__		,a__      )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class __snake_case		(      SCREAMING_SNAKE_CASE_					,	unittest.TestCase     ):
    _a		:						Tuple=				BertJapaneseTokenizer
    _a		:						Optional[Any]=				False
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           super().setUp()

           lowercase							:     Dict       	=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           lowercase							:     List[Any]       	=      os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES["""vocab_file"""]      )
           with open(self.vocab_file		,"""w"""		,encoding="""utf-8"""      ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )
    def       _SCREAMING_SNAKE_CASE       (  self		,**snake_case      ):



           '''simple docstring'''







           return BertJapaneseTokenizer.from_pretrained(self.tmpdirname		,subword_tokenizer_type="""character"""		,**a__      )
    def       _SCREAMING_SNAKE_CASE       (  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      """こんにちは、世界。 \nこんばんは、世界。"""
           lowercase							:     Optional[Any]       	=      """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
           return input_text, output_text
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           pass  # TODO add if relevant
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     int       	=      self.tokenizer_class(self.vocab_file		,subword_tokenizer_type="""character"""      )

           lowercase							:     Union[str, Any]       	=      tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""      )
           self.assertListEqual(
               a__		,["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]      )
           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(a__      )		,[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

           lowercase							:     List[Any]       	=      {}
           for i, token in enumerate(a__      ):
                  lowercase							:     int       	=      i
           lowercase							:     str       	=      CharacterTokenizer(vocab=a__		,unk_token="""[UNK]"""      )

           self.assertListEqual(tokenizer.tokenize(""""""      )		,[]      )

           self.assertListEqual(tokenizer.tokenize("""こんにちは"""      )		,["""こ""", """ん""", """に""", """ち""", """は"""]      )

           self.assertListEqual(tokenizer.tokenize("""こんにちほ"""      )		,["""こ""", """ん""", """に""", """ち""", """[UNK]"""]      )
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""      )

           lowercase							:     List[Any]       	=      tokenizer.encode("""ありがとう。"""		,add_special_tokens=a__      )
           lowercase							:     Union[str, Any]       	=      tokenizer.encode("""どういたしまして。"""		,add_special_tokens=a__      )

           lowercase							:     Optional[int]       	=      tokenizer.build_inputs_with_special_tokens(a__      )
           lowercase							:     int       	=      tokenizer.build_inputs_with_special_tokens(a__		,a__      )

           # 2 is for "[CLS]", 3 is for "[SEP]"
           assert encoded_sentence == [2] + text + [3]
           assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class __snake_case		(      unittest.TestCase     ):
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      """cl-tohoku/bert-base-japanese"""
           lowercase							:     Optional[int]       	=      AutoTokenizer.from_pretrained(a__      )
           self.assertIsInstance(a__		,a__      )






class __snake_case		(      unittest.TestCase     ):
    def       _SCREAMING_SNAKE_CASE       (  self      ):



           '''simple docstring'''







           lowercase							:     str       	=      """cl-tohoku/bert-base-japanese"""
           with self.assertLogs("""transformers"""		,level="""WARNING"""      ) as cm:
                  BertTokenizer.from_pretrained(a__      )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from."""      )      )
           lowercase							:     Dict       	=      """bert-base-cased"""
           with self.assertLogs("""transformers"""		,level="""WARNING"""      ) as cm:
                  BertJapaneseTokenizer.from_pretrained(a__      )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from."""      )      )

"""simple docstring"""




import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_a      :		Optional[int]           =   get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
@require_tokenizers
class 						__A							( SCREAMING_SNAKE_CASE_    ,	unittest.TestCase		):
     _UpperCamelCase							:			List[Any]							=		XGLMTokenizer
     _UpperCamelCase							:			List[Any]							=		XGLMTokenizerFast
     _UpperCamelCase							:			Dict							=		True
     _UpperCamelCase							:			Tuple							=		True






     def     __A  (		self  ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _lowerCAmelCase						:       List[Any]  			=     XGLMTokenizer(a__				,     keep_accents=a__  )
       tokenizer.save_pretrained(self.tmpdirname  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[str]  			=     """<pad>"""
       _lowerCAmelCase						:       List[Any]  			=     1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__  )				,     a__  )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__  )				,     a__  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       Optional[int]  			=     list(self.get_tokenizer().get_vocab().keys()  )

       self.assertEqual(vocab_keys[0]				,     """<s>"""  )
       self.assertEqual(vocab_keys[1]				,     """<pad>"""  )
       self.assertEqual(len(a__  )				,     1008  )






     def     __A  (		self  ):
       self.assertEqual(self.get_tokenizer().vocab_size				,     1008  )






     def     __A  (		self  ):
       _lowerCAmelCase						:       List[Any]  			=     XGLMTokenizer(a__				,     keep_accents=a__  )

       _lowerCAmelCase						:       Dict  			=     tokenizer.tokenize("""This is a test"""  )
       self.assertListEqual(a__				,     ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(a__  )				,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,     )

       _lowerCAmelCase						:       Any  			=     tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
       self.assertListEqual(
           a__				,     [
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]				,     )
       _lowerCAmelCase						:       List[str]  			=     tokenizer.convert_tokens_to_ids(a__  )
       self.assertListEqual(
           a__				,     [
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]				,     )

       _lowerCAmelCase						:       Optional[int]  			=     tokenizer.convert_ids_to_tokens(a__  )
       self.assertListEqual(
           a__				,     [
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]				,     )






     @cached_property
     def     __A  (		self  ):
       return XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""  )






     def     __A  (		self  ):
       with tempfile.NamedTemporaryFile() as f:
         shutil.copyfile(a__				,     f.name  )
         _lowerCAmelCase						:       Union[str, Any]  			=     XGLMTokenizer(f.name				,     keep_accents=a__  )
         _lowerCAmelCase						:       List[str]  			=     pickle.dumps(a__  )
       pickle.loads(a__  )






     def     __A  (		self  ):
       if not self.test_rust_tokenizer:
         return

       _lowerCAmelCase						:       List[str]  			=     self.get_tokenizer()
       _lowerCAmelCase						:       Optional[Any]  			=     self.get_rust_tokenizer()

       _lowerCAmelCase						:       Tuple  			=     """I was born in 92000, and this is falsé."""

       _lowerCAmelCase						:       List[Any]  			=     tokenizer.tokenize(a__  )
       _lowerCAmelCase						:       Tuple  			=     rust_tokenizer.tokenize(a__  )
       self.assertListEqual(a__				,     a__  )

       _lowerCAmelCase						:       Union[str, Any]  			=     tokenizer.encode(a__				,     add_special_tokens=a__  )
       _lowerCAmelCase						:       str  			=     rust_tokenizer.encode(a__				,     add_special_tokens=a__  )
       self.assertListEqual(a__				,     a__  )

       _lowerCAmelCase						:       int  			=     self.get_rust_tokenizer()
       _lowerCAmelCase						:       Dict  			=     tokenizer.encode(a__  )
       _lowerCAmelCase						:       List[Any]  			=     rust_tokenizer.encode(a__  )
       self.assertListEqual(a__				,     a__  )






     @slow
     def     __A  (		self  ):
       _lowerCAmelCase						:       int  			=     """Hello World!"""
       _lowerCAmelCase						:       Optional[int]  			=     [2, 31227, 4447, 35]

       self.assertListEqual(a__				,     self.big_tokenizer.encode(a__  )  )






     @slow
     def     __A  (		self  ):
       _lowerCAmelCase						:       Any  			=     (
           """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
           """ add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"""
       )
       # fmt: off
       _lowerCAmelCase						:       List[str]  			=     [2, 1018, 67, 11, 1988, 2617, 5631, 278, 11, 3407, 48, 71630, 28085, 4, 3234, 157, 13, 6, 5, 6, 4, 3526, 768, 15, 659, 57, 298, 3983, 864, 129, 21, 6, 5, 13675, 377, 652, 7580, 10341, 155, 2817, 422, 1666, 7, 1674, 53, 113, 202277, 17892, 33, 60, 87, 4, 3234, 157, 61, 2667, 52376, 19, 88, 23, 735]
       # fmt: on

       self.assertListEqual(a__				,     self.big_tokenizer.encode(a__  )  )






     @slow
     def     __A  (		self  ):
       # fmt: off
       _lowerCAmelCase						:       List[str]  			=     {
           """input_ids""": [[2, 108825, 1163, 15, 88010, 473, 15898, 157, 13672, 1857, 312, 8, 238021, 1163, 53, 13672, 1857, 312, 8, 53283, 182396, 8, 18566, 16, 36733, 4101, 8, 230, 244017, 122553, 7, 15, 132597, 4, 293, 12511, 7610, 4, 3414, 132597, 9, 4, 32361, 362, 4, 734, 28512, 32569, 18, 4, 32361, 26096, 14982, 73, 18715, 21433, 235261, 15, 492, 12427, 16, 53, 18715, 21433, 65454, 15, 23659, 563, 16, 278, 597, 2843, 595, 7931, 182396, 64186, 22, 886, 595, 132981, 53, 25540, 3449, 43982, 39901, 5951, 878, 330, 4, 27694, 80269, 312, 53, 6517, 11780, 611, 20408, 5], [2, 6, 132597, 67, 42897, 33, 592, 8, 163729, 25540, 361, 136997, 109514, 173230, 7, 501, 60, 102913, 196, 5631, 235, 63243, 473, 6, 231757, 74, 5277, 7905, 53, 3095, 37317, 22, 454, 183874, 5], [2, 268, 31298, 46530, 6, 132935, 43831, 7, 597, 32, 24, 3688, 9865, 5]],
           """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
       }  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=a__				,     model_name="""facebook/xglm-564M"""				,     padding=a__				,     )




