





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class    lowerCAmelCase							(					__UpperCamelCase,  unittest.TestCase			):
 UpperCAmelCase__													=		BertJapaneseTokenizer
 UpperCAmelCase__													=		False
 UpperCAmelCase__													=		True
 def      A_	(      self  :      Optional[int]	)			->	Dict:
     super().setUp()

     lowerCamelCase__    :      str					 =		[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     lowerCamelCase__    :      str					 =		os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['vocab_file']	)
     with open(self.vocab_file			,     'w'			,     encoding='utf-8'	) as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)
 def      A_	(      self  :      Tuple			,     UpperCAmelCase  :      Dict	)			->	Optional[int]:
     lowerCamelCase__    :      Tuple					 =		'こんにちは、世界。 \nこんばんは、世界。'
     lowerCamelCase__    :      Union[str, Any]					 =		'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text
 def      A_	(      self  :      List[str]			,     UpperCAmelCase  :      List[Any]	)			->	str:
     lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		self.get_input_output_texts(UpperCAmelCase	)
     lowerCamelCase__    :      int					 =		tokenizer.encode(UpperCAmelCase			,     add_special_tokens=UpperCAmelCase	)
     lowerCamelCase__    :      Union[str, Any]					 =		tokenizer.decode(UpperCAmelCase			,     clean_up_tokenization_spaces=UpperCAmelCase	)
     return text, ids
 def      A_	(      self  :      Dict	)			->	List[str]:
     pass  # TODO add if relevant
 def      A_	(      self  :      List[Any]	)			->	Dict:
     pass  # TODO add if relevant
 def      A_	(      self  :      Any	)			->	Dict:
     pass  # TODO add if relevant
 def      A_	(      self  :      Any	)			->	Optional[int]:
     lowerCamelCase__    :      Optional[Any]					 =		self.tokenizer_class(self.vocab_file	)

     lowerCamelCase__    :      str					 =		tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'	)
     self.assertListEqual(UpperCAmelCase			,     ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [3, 12, 10, 14, 4, 9, 12, 10, 14]	)
 def      A_	(      self  :      Any	)			->	Tuple:
     lowerCamelCase__    :      Optional[int]					 =		self.tokenizer_class(self.vocab_file			,     word_tokenizer_type='mecab'	)
     self.assertIsNotNone(UpperCAmelCase	)

     lowerCamelCase__    :      Any					 =		'こんにちは、世界。\nこんばんは、世界。'
     lowerCamelCase__    :      Union[str, Any]					 =		tokenizer.tokenize(UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

     lowerCamelCase__    :      Any					 =		os.path.join(self.tmpdirname			,     'tokenizer.bin'	)
     with open(UpperCAmelCase			,     'wb'	) as handle:
         pickle.dump(UpperCAmelCase			,     UpperCAmelCase	)

     with open(UpperCAmelCase			,     'rb'	) as handle:
         lowerCamelCase__    :      Tuple					 =		pickle.load(UpperCAmelCase	)

     lowerCamelCase__    :      List[str]					 =		tokenizer_new.tokenize(UpperCAmelCase	)

     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)
 def      A_	(      self  :      List[Any]	)			->	List[Any]:
     lowerCamelCase__    :      Union[str, Any]					 =		MecabTokenizer(mecab_dic='ipadic'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,     )
 def      A_	(      self  :      Union[str, Any]	)			->	Any:
     try:
         lowerCamelCase__    :      Optional[int]					 =		MecabTokenizer(mecab_dic='unidic_lite'	)
     except ModuleNotFoundError:
         return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,     )
 def      A_	(      self  :      int	)			->	Union[str, Any]:
     try:
         lowerCamelCase__    :      Optional[Any]					 =		MecabTokenizer(mecab_dic='unidic'	)
     except ModuleNotFoundError:
         return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,     )
 def      A_	(      self  :      str	)			->	Optional[int]:
     lowerCamelCase__    :      Union[str, Any]					 =		MecabTokenizer(do_lower_case=UpperCAmelCase			,     mecab_dic='ipadic'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,     )
 def      A_	(      self  :      int	)			->	List[str]:
     try:
         lowerCamelCase__    :      Optional[int]					 =		MecabTokenizer(
             do_lower_case=UpperCAmelCase			,     normalize_text=UpperCAmelCase			,     mecab_option='-d /usr/local/lib/mecab/dic/jumandic'	)
     except RuntimeError:
         # if dict doesn't exist in the system, previous code raises this error.
         return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']			,     )
 def      A_	(      self  :      Dict	)			->	Tuple:
     lowerCamelCase__    :      Any					 =		MecabTokenizer(normalize_text=UpperCAmelCase			,     mecab_dic='ipadic'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']			,     )
 @require_sudachi
 def      A_	(      self  :      List[Any]	)			->	Optional[Any]:
     lowerCamelCase__    :      Dict					 =		self.tokenizer_class(self.vocab_file			,     word_tokenizer_type='sudachi'	)
     self.assertIsNotNone(UpperCAmelCase	)

     lowerCamelCase__    :      Union[str, Any]					 =		'こんにちは、世界。\nこんばんは、世界。'
     lowerCamelCase__    :      List[Any]					 =		tokenizer.tokenize(UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

     lowerCamelCase__    :      Optional[Any]					 =		os.path.join(self.tmpdirname			,     'tokenizer.bin'	)
     with open(UpperCAmelCase			,     'wb'	) as handle:
         pickle.dump(UpperCAmelCase			,     UpperCAmelCase	)

     with open(UpperCAmelCase			,     'rb'	) as handle:
         lowerCamelCase__    :      Optional[int]					 =		pickle.load(UpperCAmelCase	)

     lowerCamelCase__    :      Optional[int]					 =		tokenizer_new.tokenize(UpperCAmelCase	)

     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)
 @require_sudachi
 def      A_	(      self  :      Dict	)			->	int:
     lowerCamelCase__    :      Any					 =		SudachiTokenizer(sudachi_dict_type='core'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']			,     )
 @require_sudachi
 def      A_	(      self  :      Dict	)			->	Optional[Any]:
     lowerCamelCase__    :      Optional[Any]					 =		SudachiTokenizer(sudachi_dict_type='core'			,     sudachi_split_mode='A'	)

     self.assertListEqual(tokenizer.tokenize('外国人参政権'	)			,     ['外国', '人', '参政', '権']	)
 @require_sudachi
 def      A_	(      self  :      Any	)			->	int:
     lowerCamelCase__    :      Union[str, Any]					 =		SudachiTokenizer(sudachi_dict_type='core'			,     sudachi_split_mode='B'	)

     self.assertListEqual(tokenizer.tokenize('外国人参政権'	)			,     ['外国人', '参政権']	)
 @require_sudachi
 def      A_	(      self  :      Any	)			->	Union[str, Any]:
     lowerCamelCase__    :      Dict					 =		SudachiTokenizer(sudachi_dict_type='core'			,     sudachi_split_mode='C'	)

     self.assertListEqual(tokenizer.tokenize('外国人参政権'	)			,     ['外国人参政権']	)
 @require_sudachi
 def      A_	(      self  :      str	)			->	Optional[int]:
     lowerCamelCase__    :      List[str]					 =		SudachiTokenizer(do_lower_case=UpperCAmelCase			,     sudachi_dict_type='core'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']			,     )
 @require_sudachi
 def      A_	(      self  :      Union[str, Any]	)			->	Tuple:
     lowerCamelCase__    :      int					 =		SudachiTokenizer(normalize_text=UpperCAmelCase			,     sudachi_dict_type='core'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']			,     )
 @require_sudachi
 def      A_	(      self  :      List[Any]	)			->	Tuple:
     lowerCamelCase__    :      Union[str, Any]					 =		SudachiTokenizer(trim_whitespace=UpperCAmelCase			,     sudachi_dict_type='core'	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']			,     )
 @require_jumanpp
 def      A_	(      self  :      str	)			->	List[str]:
     lowerCamelCase__    :      Union[str, Any]					 =		self.tokenizer_class(self.vocab_file			,     word_tokenizer_type='jumanpp'	)
     self.assertIsNotNone(UpperCAmelCase	)

     lowerCamelCase__    :      Any					 =		'こんにちは、世界。\nこんばんは、世界。'
     lowerCamelCase__    :      Any					 =		tokenizer.tokenize(UpperCAmelCase	)
     self.assertListEqual(UpperCAmelCase			,     ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']	)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [3, 12, 10, 14, 4, 9, 12, 10, 14]	)

     lowerCamelCase__    :      Any					 =		os.path.join(self.tmpdirname			,     'tokenizer.bin'	)
     with open(UpperCAmelCase			,     'wb'	) as handle:
         pickle.dump(UpperCAmelCase			,     UpperCAmelCase	)

     with open(UpperCAmelCase			,     'rb'	) as handle:
         lowerCamelCase__    :      Optional[int]					 =		pickle.load(UpperCAmelCase	)

     lowerCamelCase__    :      Optional[Any]					 =		tokenizer_new.tokenize(UpperCAmelCase	)

     self.assertListEqual(UpperCAmelCase			,     UpperCAmelCase	)
 @require_jumanpp
 def      A_	(      self  :      Tuple	)			->	Tuple:
     lowerCamelCase__    :      List[str]					 =		JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,     )
 @require_jumanpp
 def      A_	(      self  :      str	)			->	Optional[Any]:
     lowerCamelCase__    :      List[str]					 =		JumanppTokenizer(do_lower_case=UpperCAmelCase	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,     )
 @require_jumanpp
 def      A_	(      self  :      Tuple	)			->	List[str]:
     lowerCamelCase__    :      int					 =		JumanppTokenizer(normalize_text=UpperCAmelCase	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']			,     )
 @require_jumanpp
 def      A_	(      self  :      str	)			->	Dict:
     lowerCamelCase__    :      Optional[Any]					 =		JumanppTokenizer(trim_whitespace=UpperCAmelCase	)

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '	)			,     ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']			,     )
 @require_jumanpp
 def      A_	(      self  :      Any	)			->	Any:
     lowerCamelCase__    :      Union[str, Any]					 =		JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'	)			,     ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']			,     )
 def      A_	(      self  :      List[Any]	)			->	int:
     lowerCamelCase__    :      Optional[int]					 =		['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     lowerCamelCase__    :      int					 =		{}
     for i, token in enumerate(UpperCAmelCase	):
         lowerCamelCase__    :      List[Any]					 =		i
     lowerCamelCase__    :      int					 =		WordpieceTokenizer(vocab=UpperCAmelCase			,     unk_token='[UNK]'	)

     self.assertListEqual(tokenizer.tokenize(''	)			,     []	)

     self.assertListEqual(tokenizer.tokenize('こんにちは'	)			,     ['こんにちは']	)

     self.assertListEqual(tokenizer.tokenize('こんばんは'	)			,     ['こん', '##ばんは']	)

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'	)			,     ['こん', '##ばんは', '[UNK]', 'こんにちは']	)
 def      A_	(      self  :      Tuple	)			->	str:
     lowerCamelCase__    :      int					 =		BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'	)
     lowerCamelCase__    :      List[str]					 =		tokenizer.subword_tokenizer

     lowerCamelCase__    :      Any					 =		subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'	)
     self.assertListEqual(UpperCAmelCase			,     ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']	)

     lowerCamelCase__    :      Optional[int]					 =		subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'	)
     self.assertListEqual(UpperCAmelCase			,     ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']	)







 def      A_	(      self  :      Dict	)			->	List[Any]:
     lowerCamelCase__    :      Union[str, Any]					 =		self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'	)

     lowerCamelCase__    :      int					 =		tokenizer.encode('ありがとう。'			,     add_special_tokens=UpperCAmelCase	)
     lowerCamelCase__    :      List[str]					 =		tokenizer.encode('どういたしまして。'			,     add_special_tokens=UpperCAmelCase	)

     lowerCamelCase__    :      Optional[int]					 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase	)
     lowerCamelCase__    :      List[str]					 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,     UpperCAmelCase	)

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class    lowerCAmelCase							(					__UpperCamelCase,  unittest.TestCase			):
 UpperCAmelCase__													=		BertJapaneseTokenizer
 UpperCAmelCase__													=		False
 def      A_	(      self  :      Dict	)			->	Any:
     super().setUp()

     lowerCamelCase__    :      List[Any]					 =		['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     lowerCamelCase__    :      Any					 =		os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['vocab_file']	)
     with open(self.vocab_file			,     'w'			,     encoding='utf-8'	) as vocab_writer:
         vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)
 def      A_	(      self  :      List[Any]			,     **UpperCAmelCase  :      str	)			->	List[str]:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,     subword_tokenizer_type='character'			,     **UpperCAmelCase	)
 def      A_	(      self  :      List[str]			,     UpperCAmelCase  :      Union[str, Any]	)			->	List[Any]:
     lowerCamelCase__    :      str					 =		'こんにちは、世界。 \nこんばんは、世界。'
     lowerCamelCase__    :      List[str]					 =		'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text
 def      A_	(      self  :      Optional[Any]	)			->	List[Any]:
     pass  # TODO add if relevant
 def      A_	(      self  :      Tuple	)			->	Union[str, Any]:
     pass  # TODO add if relevant
 def      A_	(      self  :      Optional[Any]	)			->	Optional[int]:
     pass  # TODO add if relevant
 def      A_	(      self  :      Tuple	)			->	Tuple:
     lowerCamelCase__    :      Optional[int]					 =		self.tokenizer_class(self.vocab_file			,     subword_tokenizer_type='character'	)

     lowerCamelCase__    :      List[str]					 =		tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'	)
     self.assertListEqual(
         UpperCAmelCase			,     ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']	)
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(UpperCAmelCase	)			,     [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]	)
 def      A_	(      self  :      Dict	)			->	Any:
     lowerCamelCase__    :      Any					 =		['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     lowerCamelCase__    :      Optional[int]					 =		{}
     for i, token in enumerate(UpperCAmelCase	):
         lowerCamelCase__    :      Union[str, Any]					 =		i
     lowerCamelCase__    :      Optional[int]					 =		CharacterTokenizer(vocab=UpperCAmelCase			,     unk_token='[UNK]'	)

     self.assertListEqual(tokenizer.tokenize(''	)			,     []	)

     self.assertListEqual(tokenizer.tokenize('こんにちは'	)			,     ['こ', 'ん', 'に', 'ち', 'は']	)

     self.assertListEqual(tokenizer.tokenize('こんにちほ'	)			,     ['こ', 'ん', 'に', 'ち', '[UNK]']	)







 def      A_	(      self  :      Any	)			->	str:
     lowerCamelCase__    :      Dict					 =		self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'	)

     lowerCamelCase__    :      List[Any]					 =		tokenizer.encode('ありがとう。'			,     add_special_tokens=UpperCAmelCase	)
     lowerCamelCase__    :      Union[str, Any]					 =		tokenizer.encode('どういたしまして。'			,     add_special_tokens=UpperCAmelCase	)

     lowerCamelCase__    :      Tuple					 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase	)
     lowerCamelCase__    :      Optional[int]					 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			,     UpperCAmelCase	)

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class    lowerCAmelCase							(					unittest.TestCase			):
 def      A_	(      self  :      List[str]	)			->	Dict:
     lowerCamelCase__    :      Union[str, Any]					 =		'cl-tohoku/bert-base-japanese'
     lowerCamelCase__    :      int					 =		AutoTokenizer.from_pretrained(UpperCAmelCase	)
     self.assertIsInstance(UpperCAmelCase			,     UpperCAmelCase	)


class    lowerCAmelCase							(					unittest.TestCase			):
 def      A_	(      self  :      Tuple	)			->	Optional[int]:
     lowerCamelCase__    :      Union[str, Any]					 =		'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'			,     level='WARNING'	) as cm:
         BertTokenizer.from_pretrained(UpperCAmelCase	)
         self.assertTrue(
             cm.records[0].message.startswith(
                 'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                 ' is called from.'	)	)
     lowerCamelCase__    :      Optional[Any]					 =		'bert-base-cased'
     with self.assertLogs('transformers'			,     level='WARNING'	) as cm:
         BertJapaneseTokenizer.from_pretrained(UpperCAmelCase	)
         self.assertTrue(
             cm.records[0].message.startswith(
                 'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                 ' is called from.'	)	)












import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


_UpperCAmelCase				:       str	   =				pytest.mark.integration

@require_faiss
class    lowerCAmelCase							(					__UpperCamelCase			):
 def      A_	(      self  :      List[Any]	)			->	Union[str, Any]:
     lowerCamelCase__    :      List[Any]					 =		Dataset.from_dict({'filename': ['my_name-train' + '_' + str(UpperCAmelCase	) for x in np.arange(30	).tolist()]}	)
     return dset
 def      A_	(      self  :      Optional[Any]	)			->	Optional[int]:
     import faiss

     lowerCamelCase__    :      Dataset					 =		self._create_dummy_dataset()
     lowerCamelCase__    :      List[Any]					 =		dset.map(
         lambda UpperCAmelCase			,     UpperCAmelCase	: {"vecs": i * np.ones(5			,     dtype=np.floataa	)}			,     with_indices=UpperCAmelCase			,     keep_in_memory=UpperCAmelCase	)
     lowerCamelCase__    :      Tuple					 =		dset.add_faiss_index('vecs'			,     batch_size=100			,     metric_type=faiss.METRIC_INNER_PRODUCT	)
     lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		dset.get_nearest_examples('vecs'			,     np.ones(5			,     dtype=np.floataa	)	)
     self.assertEqual(examples['filename'][0]			,     'my_name-train_29'	)
     dset.drop_index('vecs'	)
 def      A_	(      self  :      Union[str, Any]	)			->	int:
     import faiss

     lowerCamelCase__    :      Dataset					 =		self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((30, 5)	) * np.arange(30	).reshape(-1			,     1	)			,     index_name='vecs'			,     batch_size=100			,     metric_type=faiss.METRIC_INNER_PRODUCT			,     )
     lowerCamelCase__ ,							lowerCamelCase__    :      Optional[Any]					 =		dset.get_nearest_examples('vecs'			,     np.ones(5			,     dtype=np.floataa	)	)
     self.assertEqual(examples['filename'][0]			,     'my_name-train_29'	)
 def      A_	(      self  :      List[str]	)			->	Tuple:
     import faiss

     lowerCamelCase__    :      Dataset					 =		self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((30, 5)	) * np.arange(30	).reshape(-1			,     1	)			,     index_name='vecs'			,     metric_type=faiss.METRIC_INNER_PRODUCT			,     )

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=UpperCAmelCase	) as tmp_file:
         dset.save_faiss_index('vecs'			,     tmp_file.name	)
         dset.load_faiss_index('vecs2'			,     tmp_file.name	)
     os.unlink(tmp_file.name	)

     lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		dset.get_nearest_examples('vecs2'			,     np.ones(5			,     dtype=np.floataa	)	)
     self.assertEqual(examples['filename'][0]			,     'my_name-train_29'	)
 def      A_	(      self  :      Any	)			->	Optional[Any]:
     lowerCamelCase__    :      Dataset					 =		self._create_dummy_dataset()
     dset.add_faiss_index_from_external_arrays(
         external_arrays=np.ones((30, 5)	) * np.arange(30	).reshape(-1			,     1	)			,     index_name='vecs'	)
     dset.drop_index('vecs'	)
     self.assertRaises(UpperCAmelCase			,     partial(dset.get_nearest_examples			,     'vecs2'			,     np.ones(5			,     dtype=np.floataa	)	)	)







 def      A_	(      self  :      Dict	)			->	Dict:
     from elasticsearch import Elasticsearch

     lowerCamelCase__    :      Dataset					 =		self._create_dummy_dataset()
     with patch('elasticsearch.Elasticsearch.search'	) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'	) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'	) as mocked_bulk:
         lowerCamelCase__    :      List[Any]					 =		{'acknowledged': True}
         mocked_bulk.return_value([(True, None)] * 30	)
         lowerCamelCase__    :      int					 =		{'hits': {'hits': [{'_score': 1, '_id': 29}]}}
         lowerCamelCase__    :      List[str]					 =		Elasticsearch()

         dset.add_elasticsearch_index('filename'			,     es_client=UpperCAmelCase	)
         lowerCamelCase__ ,							lowerCamelCase__    :      Dict					 =		dset.get_nearest_examples('filename'			,     'my_name-train_29'	)
         self.assertEqual(examples['filename'][0]			,     'my_name-train_29'	)


@require_faiss
class    lowerCAmelCase							(					__UpperCamelCase			):
 def      A_	(      self  :      Any	)			->	Dict:
     import faiss

     lowerCamelCase__    :      Tuple					 =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)

     # add vectors
     index.add_vectors(np.eye(5			,     dtype=np.floataa	)	)
     self.assertIsNotNone(index.faiss_index	)
     self.assertEqual(index.faiss_index.ntotal			,     5	)
     index.add_vectors(np.zeros((5, 5)			,     dtype=np.floataa	)	)
     self.assertEqual(index.faiss_index.ntotal			,     10	)

     # single query
     lowerCamelCase__    :      int					 =		np.zeros(5			,     dtype=np.floataa	)
     lowerCamelCase__    :      Any					 =		1
     lowerCamelCase__ ,							lowerCamelCase__    :      Union[str, Any]					 =		index.search(UpperCAmelCase	)
     self.assertRaises(UpperCAmelCase			,     index.search			,     query.reshape(-1			,     1	)	)
     self.assertGreater(scores[0]			,     0	)
     self.assertEqual(indices[0]			,     1	)

     # batched queries
     lowerCamelCase__    :      str					 =		np.eye(5			,     dtype=np.floataa	)[::-1]
     lowerCamelCase__ ,							lowerCamelCase__    :      Dict					 =		index.search_batch(UpperCAmelCase	)
     self.assertRaises(UpperCAmelCase			,     index.search_batch			,     queries[0]	)
     lowerCamelCase__    :      str					 =		[scores[0] for scores in total_scores]
     lowerCamelCase__    :      List[str]					 =		[indices[0] for indices in total_indices]
     self.assertGreater(np.min(UpperCAmelCase	)			,     0	)
     self.assertListEqual([4, 3, 2, 1, 0]			,     UpperCAmelCase	)
 def      A_	(      self  :      List[Any]	)			->	List[Any]:
     import faiss

     lowerCamelCase__    :      Any					 =		FaissIndex(string_factory='Flat'	)
     index.add_vectors(np.eye(5			,     dtype=np.floataa	)	)
     self.assertIsInstance(index.faiss_index			,     faiss.IndexFlat	)
     lowerCamelCase__    :      Tuple					 =		FaissIndex(string_factory='LSH'	)
     index.add_vectors(np.eye(5			,     dtype=np.floataa	)	)
     self.assertIsInstance(index.faiss_index			,     faiss.IndexLSH	)
     with self.assertRaises(UpperCAmelCase	):
         lowerCamelCase__    :      List[str]					 =		FaissIndex(string_factory='Flat'			,     custom_index=faiss.IndexFlat(5	)	)
 def      A_	(      self  :      List[str]	)			->	Optional[int]:
     import faiss

     lowerCamelCase__    :      Optional[Any]					 =		faiss.IndexFlat(5	)
     lowerCamelCase__    :      int					 =		FaissIndex(custom_index=UpperCAmelCase	)
     index.add_vectors(np.eye(5			,     dtype=np.floataa	)	)
     self.assertIsInstance(index.faiss_index			,     faiss.IndexFlat	)







 def      A_	(      self  :      Any	)			->	Optional[int]:
     import faiss

     lowerCamelCase__    :      Any					 =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
     index.add_vectors(np.eye(5			,     dtype=np.floataa	)	)

     # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
     # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
     # see https://bugs.python.org/issue14243 and
     # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
     with tempfile.NamedTemporaryFile(delete=UpperCAmelCase	) as tmp_file:
         index.save(tmp_file.name	)
         lowerCamelCase__    :      List[Any]					 =		FaissIndex.load(tmp_file.name	)
     os.unlink(tmp_file.name	)

     lowerCamelCase__    :      List[str]					 =		np.zeros(5			,     dtype=np.floataa	)
     lowerCamelCase__    :      Tuple					 =		1
     lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		index.search(UpperCAmelCase	)
     self.assertGreater(scores[0]			,     0	)
     self.assertEqual(indices[0]			,     1	)


@require_faiss
def        SCREAMING_SNAKE_CASE      (       _UpperCAmelCase )			->     Any:
    import faiss

    lowerCamelCase__    :      Optional[Any]					 =		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT )
    index.add_vectors(np.eye(5						,	dtype=np.floataa ) )

    lowerCamelCase__    :      Optional[int]					 =		'index.faiss'
    lowerCamelCase__    :      Optional[Any]					 =		F"""mock://{index_name}"""
    index.save(_UpperCAmelCase						,	storage_options=mockfs.storage_options )
    lowerCamelCase__    :      Tuple					 =		FaissIndex.load(_UpperCAmelCase						,	storage_options=mockfs.storage_options )

    lowerCamelCase__    :      Optional[int]					 =		np.zeros(5						,	dtype=np.floataa )
    lowerCamelCase__    :      Dict					 =		1
    lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		index.search(_UpperCAmelCase )
    assert scores[0] > 0
    assert indices[0] == 1


@require_elasticsearch
class    lowerCAmelCase							(					__UpperCamelCase			):
 def      A_	(      self  :      Dict	)			->	List[Any]:
     from elasticsearch import Elasticsearch

     with patch('elasticsearch.Elasticsearch.search'	) as mocked_search, patch(
         'elasticsearch.client.IndicesClient.create'	) as mocked_index_create, patch('elasticsearch.helpers.streaming_bulk'	) as mocked_bulk:
         lowerCamelCase__    :      Any					 =		Elasticsearch()
         lowerCamelCase__    :      Tuple					 =		{'acknowledged': True}
         lowerCamelCase__    :      Tuple					 =		ElasticSearchIndex(es_client=UpperCAmelCase	)
         mocked_bulk.return_value([(True, None)] * 3	)
         index.add_documents(['foo', 'bar', 'foobar']	)

         # single query
         lowerCamelCase__    :      Optional[int]					 =		'foo'
         lowerCamelCase__    :      str					 =		{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
         lowerCamelCase__ ,							lowerCamelCase__    :      Optional[Any]					 =		index.search(UpperCAmelCase	)
         self.assertEqual(scores[0]			,     1	)
         self.assertEqual(indices[0]			,     0	)

         # single query with timeout
         lowerCamelCase__    :      Any					 =		'foo'
         lowerCamelCase__    :      List[str]					 =		{'hits': {'hits': [{'_score': 1, '_id': 0}]}}
         lowerCamelCase__ ,							lowerCamelCase__    :      Tuple					 =		index.search(UpperCAmelCase			,     request_timeout=30	)
         self.assertEqual(scores[0]			,     1	)
         self.assertEqual(indices[0]			,     0	)

         # batched queries
         lowerCamelCase__    :      List[str]					 =		['foo', 'bar', 'foobar']
         lowerCamelCase__    :      str					 =		{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
         lowerCamelCase__ ,							lowerCamelCase__    :      str					 =		index.search_batch(UpperCAmelCase	)
         lowerCamelCase__    :      List[str]					 =		[scores[0] for scores in total_scores]
         lowerCamelCase__    :      List[Any]					 =		[indices[0] for indices in total_indices]
         self.assertGreater(np.min(UpperCAmelCase	)			,     0	)
         self.assertListEqual([1, 1, 1]			,     UpperCAmelCase	)

         # batched queries with timeout
         lowerCamelCase__    :      str					 =		['foo', 'bar', 'foobar']
         lowerCamelCase__    :      List[Any]					 =		{'hits': {'hits': [{'_score': 1, '_id': 1}]}}
         lowerCamelCase__ ,							lowerCamelCase__    :      Optional[Any]					 =		index.search_batch(UpperCAmelCase			,     request_timeout=30	)
         lowerCamelCase__    :      Optional[Any]					 =		[scores[0] for scores in total_scores]
         lowerCamelCase__    :      Dict					 =		[indices[0] for indices in total_indices]
         self.assertGreater(np.min(UpperCAmelCase	)			,     0	)
         self.assertListEqual([1, 1, 1]			,     UpperCAmelCase	)





