





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class     SCREAMING_SNAKE_CASE__ (						__snake_case				,					unittest.TestCase      ):
 __SCREAMING_SNAKE_CASE							       =	GPTSanJapaneseTokenizer
 __SCREAMING_SNAKE_CASE							       =	False
 __SCREAMING_SNAKE_CASE							       =	{'do_clean_text': False, 'add_prefix_space': False}



 def 				UpperCamelCase			(							self ):
        super().setUp()

        # fmt: off
        A__											=						["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
        # fmt: on
        A__											=						{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
        A__											=						{"unk_token": "<unk>"}

        A__											=						os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['''vocab_file'''] )
        A__											=						os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['''emoji_file'''] )
        with open(self.vocab_file,'''w''',encoding='''utf-8''' ) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
        with open(self.emoji_file,'''w''' ) as emoji_writer:
               emoji_writer.write(json.dumps(A_ ) )



 def 				UpperCamelCase			(							self,**__lowerCamelCase ):
        kwargs.update(self.special_tokens_map )
        return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname,**A_ )



 def 				UpperCamelCase			(							self,__lowerCamelCase ):
        A__											=						"こんにちは、世界。 \nこんばんは、㔺界。😀"
        A__											=						"こんにちは、世界。 \nこんばんは、世界。😀"
        return input_text, output_text



 def 				UpperCamelCase			(							self,__lowerCamelCase ):
        A__											=						self.get_input_output_texts(A_ )
        A__											=						tokenizer.encode(A_,add_special_tokens=A_ )
        A__											=						tokenizer.decode(A_,clean_up_tokenization_spaces=A_ )
        return text, ids



 def 				UpperCamelCase			(							self ):
        pass  # TODO add if relevant



 def 				UpperCamelCase			(							self ):
        pass  # TODO add if relevant



 def 				UpperCamelCase			(							self ):
        pass  # TODO add if relevant



 def 				UpperCamelCase			(							self ):
        A__											=						self.get_tokenizer()

        # Testing tokenization
        A__											=						"こんにちは、世界。　こんばんは、㔺界。"
        A__											=						["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
        A__											=						tokenizer.tokenize(A_ )
        self.assertListEqual(A_,A_ )

        # Testing conversion to ids without special tokens
        A__											=						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
        A__											=						tokenizer.convert_tokens_to_ids(A_ )
        self.assertListEqual(A_,A_ )

        # Testing conversion to ids with special tokens
        A__											=						tokens + [tokenizer.unk_token]
        A__											=						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
        A__											=						tokenizer.convert_tokens_to_ids(A_ )
        self.assertListEqual(A_,A_ )



 def 				UpperCamelCase			(							self ):
        A__											=						self.get_tokenizer()

        # Testing tokenization
        A__											=						"こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
        A__											=						"こんにちは、、、、世界。こんばんは、、、、世界。"
        A__											=						tokenizer.encode(A_ )
        A__											=						tokenizer.decode(A_ )
        self.assertEqual(A_,A_ )



 @slow
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese''' )

        # Testing tokenization
        A__											=						"こんにちは、世界。"
        A__											=						"こんばんは、㔺界。😀"
        A__											=						"こんにちは、世界。こんばんは、世界。😀"
        A__											=						tokenizer.encode(prefix_text + input_text )
        A__											=						tokenizer.encode('''''',prefix_text=prefix_text + input_text )
        A__											=						tokenizer.encode(A_,prefix_text=A_ )
        A__											=						tokenizer.decode(A_ )
        A__											=						tokenizer.decode(A_ )
        A__											=						tokenizer.decode(A_ )
        self.assertEqual(A_,A_ )
        self.assertEqual(A_,A_ )
        self.assertEqual(A_,A_ )



 @slow
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese''' )

        # Testing tokenization
        A__											=						"こんにちは、世界。"
        A__											=						"こんばんは、㔺界。😀"

        A__											=						len(tokenizer.encode(A_ ) ) - 2
        A__											=						len(tokenizer.encode(A_ ) ) - 2

        A__											=						[1] + [0] * (len_prefix + len_text + 1)
        A__											=						[1] * (len_prefix + len_text + 1) + [0]
        A__											=						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

        A__											=						tokenizer(prefix_text + input_text ).token_type_ids
        A__											=						tokenizer('''''',prefix_text=prefix_text + input_text ).token_type_ids
        A__											=						tokenizer(A_,prefix_text=A_ ).token_type_ids
        self.assertListEqual(A_,A_ )
        self.assertListEqual(A_,A_ )
        self.assertListEqual(A_,A_ )



 @slow
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese''' )

        A__											=						tokenizer.encode('''あンいワ''' )
        A__											=						tokenizer.encode('''''',prefix_text='''あンいワ''' )
        A__											=						tokenizer.encode('''いワ''',prefix_text='''あン''' )

        self.assertEqual(tokenizer.decode(A_ ),tokenizer.decode(A_ ) )
        self.assertEqual(tokenizer.decode(A_ ),tokenizer.decode(A_ ) )
        self.assertNotEqual(A_,A_ )
        self.assertNotEqual(A_,A_ )
        self.assertEqual(x_token_a[1],x_token_a[-1] )  # SEG token
        self.assertEqual(x_token_a[1],x_token_a[3] )  # SEG token



 @slow
 def 				UpperCamelCase			(							self ):
        A__											=						self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese''' )

        A__											=						[["武田信玄", "は、"], ["織田信長", "の配下の、"]]
        A__											=						tokenizer(A_,padding=A_ )
        A__											=						tokenizer.batch_encode_plus(A_,padding=A_ )

        # fmt: off
        A__											=						[[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
        A__											=						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
        A__											=						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
        # fmt: on
        self.assertListEqual(x_token.input_ids,A_ )
        self.assertListEqual(x_token.token_type_ids,A_ )
        self.assertListEqual(x_token.attention_mask,A_ )
        self.assertListEqual(x_token_a.input_ids,A_ )
        self.assertListEqual(x_token_a.token_type_ids,A_ )
        self.assertListEqual(x_token_a.attention_mask,A_ )



 def 				UpperCamelCase			(							self ):
        pass




 def 				UpperCamelCase			(							self ):
        pass












import pytest
import requests

from datasets.utils.file_utils import http_head

from .utils import OfflineSimulationMode, RequestWouldHangIndefinitelyError, offline



@pytest.mark.integration
def 		A_				(     )						-> List[Any]:
 with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT     ):
  with pytest.raises(_lowerCAmelCase     ):
   requests.request("GET"							,   "https://huggingface.co"     )
  with pytest.raises(requests.exceptions.ConnectTimeout     ):
   requests.request("GET"							,   "https://huggingface.co"							,   timeout=1.0     )



@pytest.mark.integration
def 		A_				(     )						-> Tuple:
 with offline(OfflineSimulationMode.CONNECTION_FAILS     ):
  with pytest.raises(requests.exceptions.ConnectionError     ):
   requests.request("GET"							,   "https://huggingface.co"     )



def 		A_				(     )						-> Optional[int]:
 with offline(OfflineSimulationMode.HF_DATASETS_OFFLINE_SET_TO_1     ):
  with pytest.raises(_lowerCAmelCase     ):
   http_head("https://huggingface.co"     )
