





'''simple docstring'''
import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      snake_case     (    __lowerCamelCase			,       unittest.TestCase       ):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_	:					Tuple					=GPTSanJapaneseTokenizer
 SCREAMING_SNAKE_CASE_	:					Optional[Any]					=False
 SCREAMING_SNAKE_CASE_	:					Optional[Any]					={"do_clean_text": False, "add_prefix_space": False}






 def        _lowerCamelCase					(       self       :							str	):
        super().setUp()

        # fmt: off
        __UpperCamelCase       =  ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
        # fmt: on
        __UpperCamelCase       =  {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
        __UpperCamelCase       =  {'unk_token': '<unk>'}

        __UpperCamelCase       =  os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES['vocab_file']	)
        __UpperCamelCase       =  os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES['emoji_file']	)
        with open(self.vocab_file							,	'w'							,	encoding='utf-8'	) as vocab_writer:
               vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)
        with open(self.emoji_file							,	'w'	) as emoji_writer:
               emoji_writer.write(json.dumps(__A	)	)






 def        _lowerCamelCase					(       self       :							Union[str, Any]							,	**__A       :							List[Any]	):
        kwargs.update(self.special_tokens_map	)
        return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,	**__A	)






 def        _lowerCamelCase					(       self       :							Any							,	__A       :							Dict	):
        __UpperCamelCase       =  'こんにちは、世界。 \nこんばんは、㔺界。😀'
        __UpperCamelCase       =  'こんにちは、世界。 \nこんばんは、世界。😀'
        return input_text, output_text






 def        _lowerCamelCase					(       self       :							Union[str, Any]							,	__A       :							Any	):
        __UpperCamelCase       ,						__UpperCamelCase       =  self.get_input_output_texts(__A	)
        __UpperCamelCase       =  tokenizer.encode(__A							,	add_special_tokens=__A	)
        __UpperCamelCase       =  tokenizer.decode(__A							,	clean_up_tokenization_spaces=__A	)
        return text, ids






 def        _lowerCamelCase					(       self       :							int	):
        pass  # TODO add if relevant






 def        _lowerCamelCase					(       self       :							Union[str, Any]	):
        pass  # TODO add if relevant






 def        _lowerCamelCase					(       self       :							Optional[int]	):
        pass  # TODO add if relevant






 def        _lowerCamelCase					(       self       :							Optional[int]	):
        __UpperCamelCase       =  self.get_tokenizer()

        # Testing tokenization
        __UpperCamelCase       =  'こんにちは、世界。　こんばんは、㔺界。'
        __UpperCamelCase       =  ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
        __UpperCamelCase       =  tokenizer.tokenize(__A	)
        self.assertListEqual(__A							,	__A	)

        # Testing conversion to ids without special tokens
        __UpperCamelCase       =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
        __UpperCamelCase       =  tokenizer.convert_tokens_to_ids(__A	)
        self.assertListEqual(__A							,	__A	)

        # Testing conversion to ids with special tokens
        __UpperCamelCase       =  tokens + [tokenizer.unk_token]
        __UpperCamelCase       =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
        __UpperCamelCase       =  tokenizer.convert_tokens_to_ids(__A	)
        self.assertListEqual(__A							,	__A	)






 def        _lowerCamelCase					(       self       :							Dict	):
        __UpperCamelCase       =  self.get_tokenizer()

        # Testing tokenization
        __UpperCamelCase       =  'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
        __UpperCamelCase       =  'こんにちは、、、、世界。こんばんは、、、、世界。'
        __UpperCamelCase       =  tokenizer.encode(__A	)
        __UpperCamelCase       =  tokenizer.decode(__A	)
        self.assertEqual(__A							,	__A	)






 @slow
 def        _lowerCamelCase					(       self       :							Any	):
        __UpperCamelCase       =  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'	)

        # Testing tokenization
        __UpperCamelCase       =  'こんにちは、世界。'
        __UpperCamelCase       =  'こんばんは、㔺界。😀'
        __UpperCamelCase       =  'こんにちは、世界。こんばんは、世界。😀'
        __UpperCamelCase       =  tokenizer.encode(prefix_text + input_text	)
        __UpperCamelCase       =  tokenizer.encode(''							,	prefix_text=prefix_text + input_text	)
        __UpperCamelCase       =  tokenizer.encode(__A							,	prefix_text=__A	)
        __UpperCamelCase       =  tokenizer.decode(__A	)
        __UpperCamelCase       =  tokenizer.decode(__A	)
        __UpperCamelCase       =  tokenizer.decode(__A	)
        self.assertEqual(__A							,	__A	)
        self.assertEqual(__A							,	__A	)
        self.assertEqual(__A							,	__A	)






 @slow
 def        _lowerCamelCase					(       self       :							Optional[Any]	):
        __UpperCamelCase       =  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'	)

        # Testing tokenization
        __UpperCamelCase       =  'こんにちは、世界。'
        __UpperCamelCase       =  'こんばんは、㔺界。😀'

        __UpperCamelCase       =  len(tokenizer.encode(__A	)	) - 2
        __UpperCamelCase       =  len(tokenizer.encode(__A	)	) - 2

        __UpperCamelCase       =  [1] + [0] * (len_prefix + len_text + 1)
        __UpperCamelCase       =  [1] * (len_prefix + len_text + 1) + [0]
        __UpperCamelCase       =  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

        __UpperCamelCase       =  tokenizer(prefix_text + input_text	).token_type_ids
        __UpperCamelCase       =  tokenizer(''							,	prefix_text=prefix_text + input_text	).token_type_ids
        __UpperCamelCase       =  tokenizer(__A							,	prefix_text=__A	).token_type_ids
        self.assertListEqual(__A							,	__A	)
        self.assertListEqual(__A							,	__A	)
        self.assertListEqual(__A							,	__A	)






 @slow
 def        _lowerCamelCase					(       self       :							Any	):
        __UpperCamelCase       =  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'	)

        __UpperCamelCase       =  tokenizer.encode('あンいワ'	)
        __UpperCamelCase       =  tokenizer.encode(''							,	prefix_text='あンいワ'	)
        __UpperCamelCase       =  tokenizer.encode('いワ'							,	prefix_text='あン'	)

        self.assertEqual(tokenizer.decode(__A	)							,	tokenizer.decode(__A	)	)
        self.assertEqual(tokenizer.decode(__A	)							,	tokenizer.decode(__A	)	)
        self.assertNotEqual(__A							,	__A	)
        self.assertNotEqual(__A							,	__A	)
        self.assertEqual(x_token_a[1]							,	x_token_a[-1]	)  # SEG token
        self.assertEqual(x_token_a[1]							,	x_token_a[3]	)  # SEG token






 @slow
 def        _lowerCamelCase					(       self       :							List[str]	):
        __UpperCamelCase       =  self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'	)

        __UpperCamelCase       =  [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
        __UpperCamelCase       =  tokenizer(__A							,	padding=__A	)
        __UpperCamelCase       =  tokenizer.batch_encode_plus(__A							,	padding=__A	)

        # fmt: off
        __UpperCamelCase       =  [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
        __UpperCamelCase       =  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
        __UpperCamelCase       =  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
        # fmt: on
        self.assertListEqual(x_token.input_ids							,	__A	)
        self.assertListEqual(x_token.token_type_ids							,	__A	)
        self.assertListEqual(x_token.attention_mask							,	__A	)
        self.assertListEqual(x_token_a.input_ids							,	__A	)
        self.assertListEqual(x_token_a.token_type_ids							,	__A	)
        self.assertListEqual(x_token_a.attention_mask							,	__A	)






 def        _lowerCamelCase					(       self       :							str	):
        # Intentionally convert some words to accommodate character fluctuations unique to Japanese
        pass






 def        _lowerCamelCase					(       self       :							str	):
        # tokenizer has no padding token
        pass







'''simple docstring'''
# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


a__							:							Tuple ='''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
    import torch

    print('''Torch version:''', torch.__version__)
    print('''Cuda available:''', torch.cuda.is_available())
    print('''Cuda version:''', torch.version.cuda)
    print('''CuDNN version:''', torch.backends.cudnn.version())
    print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
    print('''Torch version:''', None)

try:
    import transformers

    print('''transformers version:''', transformers.__version__)
except ImportError:
    print('''transformers version:''', None)
