


"""simple docstring"""






import unittest

import numpy as np

from transformers import (
    GPTSanJapaneseConfig,
    GPTSanJapaneseForConditionalGeneration,
    GPTSanJapaneseModel,
    GPTSanJapaneseTokenizer,
    is_torch_available,
)
from transformers.generation import GenerationConfig
from transformers.testing_utils import require_torch, slow, tooslow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


class 			SCREAMING_SNAKE_CASE_  :




							'''simple docstring'''





							def __init__(						self	,					lowerCamelCase__	,					lowerCamelCase__=36_000	,					lowerCamelCase__=13	,					lowerCamelCase__=7	,					lowerCamelCase__=True	,					lowerCamelCase__=32	,					lowerCamelCase__=42	,					lowerCamelCase__=5	,					lowerCamelCase__=2	,					lowerCamelCase__=4	,					lowerCamelCase__=2	,					lowerCamelCase__=32	,					lowerCamelCase__=80	,					lowerCamelCase__=33	,					lowerCamelCase__=0.0	,					lowerCamelCase__=1E-6	,					lowerCamelCase__=100	,					lowerCamelCase__=0.0	,					)   ->	List[str]:







								'''simple docstring'''




								snake_case__  :						Any		=      vocab_size
								snake_case__  :						List[Any]		=      parent
								snake_case__  :						int		=      batch_size
								snake_case__  :						Optional[Any]		=      num_contexts
								# For common tests
								snake_case__  :						Optional[Any]		=      self.num_contexts
								snake_case__  :						Tuple		=      is_training
								snake_case__  :						str		=      hidden_size
								snake_case__  :						Union[str, Any]		=      num_ext_layers
								snake_case__  :						Optional[Any]		=      ext_size
								snake_case__  :						Optional[Any]		=      num_hidden_layers
								snake_case__  :						Union[str, Any]		=      num_attention_heads
								snake_case__  :						Union[str, Any]		=      num_experts
								snake_case__  :						Dict		=      d_ff
								snake_case__  :						Optional[int]		=      d_ext
								snake_case__  :						Any		=      d_spout
								snake_case__  :						str		=      dropout_rate
								snake_case__  :						Tuple		=      layer_norm_epsilon
								snake_case__  :						Optional[int]		=      expert_capacity
								snake_case__  :						str		=      router_jitter_noise




							def        UpperCAmelCase (						self)   ->	Any:







								'''simple docstring'''




								return GPTSanJapaneseConfig.from_pretrained("Tanrei/GPTSAN-japanese")




							def        UpperCAmelCase (						self)   ->	Tuple:







								'''simple docstring'''




								snake_case__  :						Union[str, Any]		=      ids_tensor([self.batch_size, self.seq_length]	,					self.vocab_size)

								snake_case__  :						List[str]		=      self.get_config()

								return (config, input_ids)




							def        UpperCAmelCase (						self)   ->	Optional[Any]:







								'''simple docstring'''




								snake_case__  :						int		=      ids_tensor([self.batch_size, self.seq_length]	,					self.vocab_size)

								snake_case__  :						List[Any]		=      self.get_config()

								return (config, {"input_ids": input_ids})




							def        UpperCAmelCase (						self)   ->	Dict:







								'''simple docstring'''




								return GPTSanJapaneseConfig(
								    vocab_size=self.vocab_size	,					num_contexts=self.seq_length	,					d_model=self.hidden_size	,					d_ff=self.d_ff	,					d_ext=self.d_ext	,					d_spout=self.d_spout	,					num_switch_layers=self.num_hidden_layers - self.num_ext_layers	,					num_ext_layers=self.num_ext_layers	,					num_heads=self.num_attention_heads	,					num_experts=self.num_experts	,					expert_capacity=self.expert_capacity	,					dropout_rate=self.dropout_rate	,					layer_norm_epsilon=self.layer_norm_epsilon	,					router_jitter_noise=self.router_jitter_noise	,					)







							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__	,					)   ->	Tuple:







								'''simple docstring'''




								snake_case__  :						str		=      GPTSanJapaneseForConditionalGeneration(config=lowerCamelCase__)
								model.to(lowerCamelCase__)
								model.eval()
								snake_case__  :						Any		=      model(
								    input_ids=lowerCamelCase__	,					)
								self.parent.assertIsNotNone(lowerCamelCase__)


@require_torch
class 			SCREAMING_SNAKE_CASE_  (   _lowercase	,      _lowercase	,      unittest.TestCase):




							'''simple docstring'''





							__magic_name__					:     Optional[int]				  =  (GPTSanJapaneseModel,) if is_torch_available() else ()
							__magic_name__					:     Any				  =  (
							    {
							        '''conversational''': GPTSanJapaneseForConditionalGeneration,
							        '''feature-extraction''': GPTSanJapaneseForConditionalGeneration,
							        '''summarization''': GPTSanJapaneseForConditionalGeneration,
							        '''text2text-generation''': GPTSanJapaneseForConditionalGeneration,
							        '''translation''': GPTSanJapaneseForConditionalGeneration,
							    }
							    if is_torch_available()
							    else {}
							)
							__magic_name__					:     Optional[Any]				  =  False
							__magic_name__					:     Union[str, Any]				  =  False
							__magic_name__					:     Tuple				  =  False
							__magic_name__					:     Any				  =  False
							__magic_name__					:     Any				  =  False
							__magic_name__					:     Optional[int]				  =  False
							__magic_name__					:     List[Any]				  =  False
							__magic_name__					:     List[Any]				  =  False
							# The small GPTSAN_JAPANESE model needs higher percentages for CPU/MP tests
							__magic_name__					:     List[str]				  =  [0.8, 0.9]




							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__)   ->	Optional[Any]:







								'''simple docstring'''




								if pipeline_test_casse_name == "SummarizationPipelineTests":
									# TODO: fix `_reorder_cache` is not implemented for this model
									return True
								elif pipeline_test_casse_name == "Text2TextGenerationPipelineTests":
									# TODO: check this.
									return True

								return False




							def        UpperCAmelCase (						self)   ->	Any:







								'''simple docstring'''




								snake_case__  :						Optional[Any]		=      GPTSanJapaneseTester(self)
								snake_case__  :						Optional[Any]		=      ConfigTester(self	,					config_class=lowerCamelCase__	,					d_model=37)




							def        UpperCAmelCase (						self)   ->	List[str]:







								'''simple docstring'''




								GPTSanJapaneseConfig()




							def        UpperCAmelCase (						self)   ->	str:







								'''simple docstring'''




								snake_case__  :						Tuple		=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCamelCase__)




							@unittest.skip(
							    reason="skip for now as the computed `max_memory` by `model_split_percents` in the test method will be changed inside `from_pretrained`")
							def        UpperCAmelCase (						self)   ->	Optional[Any]:







								'''simple docstring'''




								super().test_model_parallelism()







							@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests.")
							def        UpperCAmelCase (						self)   ->	int:







								'''simple docstring'''




								pass


@require_torch
class 			SCREAMING_SNAKE_CASE_  (   _lowercase	,      _lowercase	,      unittest.TestCase):




							'''simple docstring'''





							__magic_name__					:     List[str]				  =  (GPTSanJapaneseForConditionalGeneration,) if is_torch_available() else ()
							__magic_name__					:     Dict				  =  False
							__magic_name__					:     List[str]				  =  False
							__magic_name__					:     List[Any]				  =  False
							__magic_name__					:     int				  =  False
							__magic_name__					:     Union[str, Any]				  =  False
							__magic_name__					:     Union[str, Any]				  =  False
							# The small GPTSAN_JAPANESE model needs higher percentages for CPU/MP tests
							__magic_name__					:     Optional[int]				  =  [0.8, 0.9]




							def        UpperCAmelCase (						self)   ->	List[str]:







								'''simple docstring'''




								snake_case__  :						Optional[int]		=      GPTSanJapaneseTester(self)
								snake_case__  :						List[str]		=      ConfigTester(self	,					config_class=lowerCamelCase__	,					d_model=37)




							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								GPTSanJapaneseConfig()




							def        UpperCAmelCase (						self)   ->	Any:







								'''simple docstring'''




								snake_case__  :						str		=      self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCamelCase__)




							@unittest.skip(
							    reason="skip for now as the computed `max_memory` by `model_split_percents` in the test method will be changed inside `from_pretrained`")
							def        UpperCAmelCase (						self)   ->	List[Any]:







								'''simple docstring'''




								super().test_model_parallelism()




							@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests.")
							def        UpperCAmelCase (						self)   ->	List[str]:







								'''simple docstring'''




								pass




							@slow
							def        UpperCAmelCase (						self)   ->	Any:







								'''simple docstring'''




								snake_case__  :						Union[str, Any]		=      GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						str		=      GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Union[str, Any]		=      tokenizer.encode("武田信玄は"	,					return_tensors="pt")
								snake_case__  :						Any		=      model(lowerCamelCase__)
								snake_case__  :						Tuple		=      outputs.logits.detach().cpu().numpy()
								# Output of original model created with mesh-tensoflow
								snake_case__  :						Optional[int]		=      [
								    # fmt: off
								    [-12.0_37_83_98_89_52_63_67, -12.4_33_06_15_99_73_14_45, -14.3_33_84_03_70_17_82_23, -12.4_50_34_59_93_04_19_92, -11.1_66_13_76_95_31_25,
								    -11.9_30_13_76_34_27_73_44, -10.6_59_74_04_47_99_80_47, -12.9_09_57_45_08_66_69_92, -13.2_41_04_30_90_82_03_12, -13.3_98_57_95_97_47_31_45,
								    -11.1_07_52_48_71_82_61_72, -12.3_68_59_41_69_61_67, -22.97_94_31_15_23_43_75, -10.4_81_06_76_57_47_07_03, -12.4_84_03_07_23_57_17_77,
								    -12.8_07_36_06_49_10_88_87, -14.7_69_70_00_50_35_40_04, -12.2_33_57_96_35_62_01_17, -13.4_28_14_54_08_63_03_71, -22.6_24_17_79_32_73_92_58],
								    [-7.5_11_14_98_83_27_02_64, -8.2_81_85_17_68_49_36_52, -7.9_43_12_71_55_30_39_55, -7.55_02_13_33_69_44_58, -6.49_86_99_66_50_69_58,
								    -7.5_86_79_62_83_72_19_24, -6.9_78_08_59_94_72_04_59, -7.8_39_14_51_83_56_32_32, -8.21_96_41_68_54_85_84, -8.6_95_09_12_47_55_85_94,
								    -6.7_06_91_06_10_19_89_75, -6.65_85_79_82_63_54_98_05, -19.5_65_69_86_23_65_72_27, -5.3_53_84_27_35_29_05_27, -8.3_50_68_60_73_30_32_23,
								    -8.0_39_38_86_56_61_62_11, -10.8_56_56_92_90_16_11_33, -7.75_15_44_47_55_55_42, -8.8_19_02_21_78_64_99_02, -19.51_53_27_45_36_13_28],
								    [-9.73_06_62_34_58_86_23, -10.2_23_92_27_29_49_21_88, -9.9_32_98_14_91_08_88_67, -11.8_57_83_67_23_32_76_37, -7.6_62_62_62_66_47_94_92,
								    -11.13_52_91_09_95_48_34, -7.7_65_09_76_18_10_30_27, -11.4_72_92_32_78_80_85_94, -9.5_43_14_99_48_12_01_17, -11.9_05_63_39_26_39_16_02,
								    -9.3_66_16_42_07_45_84_96, -11.5_73_42_81_53_99_17, -23.6_99_00_32_19_60_44_92, -9.4_29_59_02_25_21_97_27, -10.42_83_92_41_02_78_32,
								    -10.5_85_24_03_64_07_47_07, -10.94_77_19_57_39_74_61, -11.0_95_41_60_69_03_07_62, -10.3_90_24_06_69_25_04_88, -23.7_69_37_29_40_06_34_77],
								    [-9.7_28_26_57_62_32_91_02, -9.8_59_71_26_00_70_80_08, -10.09_72_92_90_00_85_45, -9.6_78_52_21_09_98_53_52, -6.8_79_51_99_39_42_26_07,
								    -9.68_48_75_48_82_81_25, -4.28_03_42_57_88_87_93_95, -10.0_18_91_42_22_71_72_85, -9.3_08_44_59_30_48_09_57, -10.63_39_45_46_50_87_89,
								    -8.0_83_64_67_74_29_19_92, -9.06_30_14_98_41_30_86, -21.9_04_26_63_57_42_18_75, -8.90_16_08_46_71_02_05, -8.8_41_87_60_29_96_82_62,
								    -11.8_56_71_99_70_70_31_25, -12.0_79_39_81_55_21_24_02, -11.2_33_75_32_04_34_57_03, -10.1_77_33_86_00_15_86_91, -21.87_25_66_22_31_44_53],
								    [-9.6_69_76_45_18_73_77_93, -9.6_14_19_86_84_69_23_83, -9.8_14_51_03_45_45_89_84, -9.9_96_50_19_22_60_74_22, -11.3_75_69_04_60_20_50_78,
								    -10.1_13_40_52_27_66_11_33, -10.5_46_86_73_70_60_54_69, -10.04_36_90_68_14_57_52, -10.9_07_80_92_57_50_73_24, -10.5_04_21_61_94_15_28_32,
								    -11.1_29_19_90_28_01_51_37, -10.1_51_12_40_00_54_93_16, -21.96_58_67_99_62_15_82, -9.0_86_34_94_87_30_46_88, -11.7_30_33_90_50_29_29_69,
								    -10.4_60_66_76_10_16_84_57, -10.2_98_04_99_26_75_78_12, -10.7_84_14_82_16_24_75_59, -10.8_40_69_34_73_81_59_18, -22.03_15_28_47_29_00_39],
								    # fmt: on
								]
								snake_case__  :						Tuple		=      np.array(lowerCamelCase__).flatten()
								snake_case__  :						List[str]		=      output_logits[0, :, :20].flatten()

								def check(lowerCamelCase__	,					lowerCamelCase__	,					lowerCamelCase__=5E-4):
									return abs(a - b) < epsilon * max(abs(lowerCamelCase__)	,					abs(lowerCamelCase__))

								self.assertTrue(np.all([check(target[i]	,					predict[i]) for i in range(len(lowerCamelCase__))]))




							@slow
							def        UpperCAmelCase (						self)   ->	int:







								'''simple docstring'''




								snake_case__  :						Any		=      GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Optional[Any]		=      GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese")
								model.to(lowerCamelCase__)

								# set deterministically
								snake_case__  :						List[Any]		=      GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Dict		=      1

								# use different length sentences to test batching
								snake_case__  :						Union[str, Any]		=      [
								    "甲斐なら武田と言うほど",
								    "織田信長は、",
								]

								snake_case__  :						Optional[Any]		=      "left"
								snake_case__  :						Dict		=      tokenizer(lowerCamelCase__	,					return_tensors="pt"	,					padding=lowerCamelCase__)
								snake_case__  :						str		=      inputs["input_ids"].to(lowerCamelCase__)

								self.assertNotEqual(inputs["attention_mask"][0].numpy().tolist()	,					inputs["attention_mask"][1].numpy().tolist())

								snake_case__  :						List[str]		=      model.generate(
								    input_ids=lowerCamelCase__	,					attention_mask=inputs["attention_mask"].to(lowerCamelCase__)	,					max_new_tokens=3	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						List[Any]		=      tokenizer(sentences[0]	,					return_tensors="pt").input_ids.to(lowerCamelCase__)
								snake_case__  :						Optional[int]		=      model.generate(
								    input_ids=lowerCamelCase__	,					max_new_tokens=3	,					generation_config=lowerCamelCase__)

								snake_case__  :						Optional[int]		=      tokenizer(sentences[1]	,					return_tensors="pt").input_ids.to(lowerCamelCase__)
								snake_case__  :						Any		=      model.generate(input_ids=lowerCamelCase__	,					max_new_tokens=3	,					generation_config=lowerCamelCase__)

								self.assertNotEqual(inputs_non_padded.shape	,					inputs_padded.shape)

								snake_case__  :						int		=      tokenizer.batch_decode(lowerCamelCase__	,					skip_special_tokens=lowerCamelCase__)
								snake_case__  :						Any		=      tokenizer.decode(output_non_padded[0]	,					skip_special_tokens=lowerCamelCase__)
								snake_case__  :						Any		=      tokenizer.decode(output_padded[0]	,					skip_special_tokens=lowerCamelCase__)

								snake_case__  :						Optional[int]		=      [
								    "甲斐なら武田と言うほど甲斐の武田",
								    "織田信長は、このような",
								]
								self.assertListEqual(lowerCamelCase__	,					lowerCamelCase__)
								self.assertListEqual(lowerCamelCase__	,					[non_padded_sentence, padded_sentence])




							@tooslow
							def        UpperCAmelCase (						self)   ->	List[Any]:







								'''simple docstring'''




								snake_case__  :						int		=      GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						List[Any]		=      GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese")
								# Output of original model created with mesh-tensoflow
								snake_case__  :						Dict		=      [
								    ("武田信玄は", 35_675),
								    ("武田信玄は、", 45),
								    ("武田信玄は、この", 29),
								    ("武田信玄は、このよう", 30_642),
								    ("武田信玄は、このような", 35_680),
								    ("武田信玄は、このような「", 8_640),
								    ("武田信玄は、このような「武田", 31_617),
								    ("武田信玄は、このような「武田家", 30_646),
								    ("武田信玄は、このような「武田家の", 31_617),
								    ("武田信玄は、このような「武田家の家", 31_381),
								]
								for input, output in target:
									snake_case__  :						Union[str, Any]		=      tokenizer.encode(lowerCamelCase__	,					return_tensors="pt")
									snake_case__  :						List[Any]		=      model(lowerCamelCase__)
									snake_case__  :						Optional[Any]		=      outputs.logits.detach().cpu().numpy()[0]
									snake_case__  :						Optional[int]		=      np.argmax(output_logits[-1])
									self.assertEqual(lowerCamelCase__	,					lowerCamelCase__)




							@slow
							def        UpperCAmelCase (						self)   ->	int:







								'''simple docstring'''




								snake_case__  :						int		=      GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Union[str, Any]		=      GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese")
								model.to(lowerCamelCase__)

								# set deterministically
								snake_case__  :						Optional[Any]		=      GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Optional[Any]		=      1

								snake_case__  :						List[Any]		=      "武田信玄は、"
								snake_case__  :						Union[str, Any]		=      tokenizer(lowerCamelCase__	,					return_tensors="pt").input_ids.to(lowerCamelCase__)
								snake_case__  :						List[Any]		=      tokenizer([input_text, input_text]	,					return_tensors="pt").input_ids.to(lowerCamelCase__)

								# spout from uniform and one-hot
								snake_case__  :						List[str]		=      [
								    # fmt: off
								    [0.87_88_22_08, 0.38_42_63_96, 0.33_22_02_48, 0.43_89_04_06, 0.16_56_22_52,
								    0.04_80_39_85, 0.21_15_72  , 0.23_18_84_73, 0.37_15_30_68, 0.7_83_63_77 ,
								    0.02_16_01_72, 0.38_76_17_19, 0.75_29_07_72, 0.90_19_88_57, 0.34_36_57_77,
								    0.64_16_81_69, 0.44_31_84_71, 0.14_57_57_46, 0.92_56_28_81, 0.40_81_21_48,
								    0.29_01_91_22, 0.88_86_15_99, 0.65_52_48_46, 0.43_56_34_56, 0.38_17_71_87,
								    0.70_83_29_65, 0.81_52_78_92, 0.68_83_28_12, 0.38_83_31_92, 0.4_56_15_22 ,
								    0.14_82_88_17, 0.47_24_82_13, 0.54_35_73_35, 0.82_00_95_66, 0.1_33_88_84 ,
								    0.02_75_54_17, 0.19_76_46_77, 0.2_42_20_84 , 0.04_75_76_74, 0.65_40_96_06,
								    0.0_82_45_89 , 0.03_30_43_83, 0.94_38_76_89, 0.98_76_45_09, 0.82_43_39_01,
								    0.27_64_67_41, 0.64_90_74_93, 0.76_00_94_06, 0.30_08_79_15, 0.17_90_46_89,
								    0.41_60_17_14, 0.67_04_63_98, 0.10_42_28_22, 0.08_44_73_74, 0.07_35_43_44,
								    0.61_42_35_65, 0.70_28_48_66, 0.7_53_23_33 , 0.1_97_20_38 , 0.29_57_56_59,
								    0.90_58_38_86, 0.29_26_53_07, 0.50_00_01_75, 0.70_40_76_55, 0.88_93_63  ,
								    0.81_90_44_18, 0.66_82_91_28, 0.64_46_88_15, 0.56_56_37_23, 0.85_60_18_75,
								    0.94_92_46_72, 0.00_16_67_62, 0.25_22_06_43, 0.74_54_02_19, 0.67_99_32_47,
								    0.1_54_96_75 , 0.39_38_53_52, 0.92_15_36_07, 0.63_74_59_31, 0.27_75_90_43,
								    0.84_70_22_95, 0.65_90_42_71, 0.58_67_66_14, 0.8_66_69_36 , 0.39_60_74_38,
								    0.79_95_49_83, 0.42_22_06_97, 0.39_65_03_81, 0.7_84_98_64 , 0.56_15_02_01,
								    0.15_67_89_25, 0.14_74_60_32, 0.34_54_21_14, 0.47_02_67_83, 0.11_95_64_89,
								    0.25_42_14_35, 0.33_78_89_01, 0.68_93_48_42, 0.36_42_46_85, 0.71_73_78_98,
								    0.38_98_34_49, 0.94_39_37_79, 0.39_57_55_88, 0.36_61_65_53, 0.87_10_46_65,
								    0.64_63_02_03, 0.22_51_69_05, 0.88_27_08_04, 0.15_03_13_38, 0.75_14_43_45,
								    0.46_45_90_25, 0.85_39_64_54, 0.86_35_56_43, 0.65_13_98_51, 0.70_26_60_61,
								    0.30_24_13_89, 0.81_05_64_97, 0.88_86_59_69, 0.38_77_38_07, 0.70_63_58_49,
								    0.90_71_84_59, 0.43_24_57_89, 0.28_00_06_54, 0.45_93_55_62, 0.08_77_35_19,
								    0.9_55_21_51 , 0.93_90_15_11, 0.22_48_92_88], # uniform
								    [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
								     0., 0., 0., 0., 0., 0., 0., 0.],
								    # fmt: on
								]

								snake_case__  :						Optional[Any]		=      model.generate(
								    input_ids=lowerCamelCase__	,					spout=spouts[0]	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						Optional[int]		=      model.generate(
								    input_ids=lowerCamelCase__	,					spout=spouts[1]	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						str		=      model.generate(
								    input_ids=lowerCamelCase__	,					spout=lowerCamelCase__	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						Tuple		=      tokenizer.decode(outputa[0])
								snake_case__  :						Dict		=      tokenizer.decode(outputa[0])
								snake_case__  :						Tuple		=      tokenizer.batch_decode(lowerCamelCase__)

								snake_case__  :						int		=      [
								    "武田信玄は、武田氏の滅亡後、武田氏の居城であった甲斐武田氏の居城である",
								    "武田信玄は、武田家の滅亡を防ぐため、武田家の家臣である武田信虎を討",
								]
								self.assertListEqual(lowerCamelCase__	,					lowerCamelCase__)
								self.assertListEqual(lowerCamelCase__	,					[outa_sentence, outa_sentence])







							@slow
							def        UpperCAmelCase (						self)   ->	Optional[Any]:







								'''simple docstring'''




								snake_case__  :						List[Any]		=      GPTSanJapaneseForConditionalGeneration.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Tuple		=      GPTSanJapaneseTokenizer.from_pretrained("Tanrei/GPTSAN-japanese")
								model.to(lowerCamelCase__)

								# set deterministically
								snake_case__  :						Dict		=      GenerationConfig.from_pretrained("Tanrei/GPTSAN-japanese")
								snake_case__  :						Tuple		=      1

								snake_case__  :						Union[str, Any]		=      "武田信玄"
								snake_case__  :						str		=      "織田信長"
								snake_case__  :						Any		=      "は、"
								snake_case__  :						Optional[int]		=      "が、"
								snake_case__  :						str		=      tokenizer(lowerCamelCase__	,					prefix_text=lowerCamelCase__	,					return_tensors="pt")
								snake_case__  :						List[str]		=      tokenizer(lowerCamelCase__	,					prefix_text=lowerCamelCase__	,					return_tensors="pt")
								snake_case__  :						List[Any]		=      tokenizer([[prefix_text_a, input_text_a], [prefix_text_a, input_text_a]]	,					return_tensors="pt")

								snake_case__  :						int		=      model.generate(
								    input_ids=input_tok_a.input_ids.to(lowerCamelCase__)	,					token_type_ids=input_tok_a.token_type_ids.to(lowerCamelCase__)	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						Optional[Any]		=      model.generate(
								    input_ids=input_tok_a.input_ids.to(lowerCamelCase__)	,					token_type_ids=input_tok_a.token_type_ids.to(lowerCamelCase__)	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						Any		=      model.generate(
								    input_ids=input_tok_a.input_ids.to(lowerCamelCase__)	,					token_type_ids=input_tok_a.token_type_ids.to(lowerCamelCase__)	,					attention_mask=input_tok_a.attention_mask.to(lowerCamelCase__)	,					max_new_tokens=20	,					generation_config=lowerCamelCase__	,					)

								snake_case__  :						List[str]		=      tokenizer.decode(outputa[0])
								snake_case__  :						Union[str, Any]		=      tokenizer.decode(outputa[0])
								snake_case__  :						List[Any]		=      tokenizer.batch_decode(lowerCamelCase__)

								snake_case__  :						Union[str, Any]		=      [
								    "武田信玄は、武田氏の祖である武田信虎を、その子・武田信友を擁して",
								    "織田信長が、織田信長の妻・お市の方を妻として迎えたという逸話が残",
								]
								self.assertListEqual(lowerCamelCase__	,					lowerCamelCase__)
								self.assertListEqual(lowerCamelCase__	,					[outa_sentence, outa_sentence])







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
	if is_torch_available():
		import torch
	if is_tf_available():
		import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


__a							  =						logging.get_logger(__name__)

__a							  =						{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

__a							  =						{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

__a							  =						{
    """Salesforce/codegen-350M-mono""": 2_0_4_8,
}

@lru_cache()
def    UpperCamelCase_ (	)     ->List[Any]:
		A				=(
		    list(range(ord("!"		)  ,				ord("~"		) + 1		)		) + list(range(ord("¡"		)  ,				ord("¬"		) + 1		)		) + list(range(ord("®"		)  ,				ord("ÿ"		) + 1		)		)
		)
		A				=bs[:]
		A				=0
		for b in range(2**8		):
				if b not in bs:
						bs.append(a_		)
						cs.append(2**8 + n		)
						n += 1
		A				=[chr(a_		) for n in cs]
		return dict(zip(a_  ,				a_		)		)

def    UpperCamelCase_ (	a_		)     ->Union[str, Any]:
		A				=set()
		A				=word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)		)
				A				=char
		return pairs






class        UpperCamelCase__(    lowerCAmelCase__	):

	"""simple docstring"""

	_A  =		VOCAB_FILES_NAMES
	_A  =		PRETRAINED_VOCAB_FILES_MAP
	_A  =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A  =		["input_ids", "attention_mask"]







	def __init__(      self				:			Optional[int]							,			snake_case__				:			List[Any]							,			snake_case__				:			int							,			snake_case__				:			Tuple="replace"							,			snake_case__				:			List[str]="<|endoftext|>"							,			snake_case__				:			Tuple="<|endoftext|>"							,			snake_case__				:			Union[str, Any]="<|endoftext|>"							,			snake_case__				:			int=None							,			snake_case__				:			Optional[int]=False							,			snake_case__				:			Tuple=False							,			**snake_case__				:			Optional[Any]							,			):
			"""simple docstring"""





			A				=AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__ ) if isinstance(snake_case__							,			snake_case__ ) else bos_token
			A				=AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__ ) if isinstance(snake_case__							,			snake_case__ ) else eos_token
			A				=AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__ ) if isinstance(snake_case__							,			snake_case__ ) else unk_token
			A				=AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__ ) if isinstance(snake_case__							,			snake_case__ ) else pad_token
			super().__init__(
			    errors=snake_case__							,			unk_token=snake_case__							,			bos_token=snake_case__							,			eos_token=snake_case__							,			pad_token=snake_case__							,			add_prefix_space=snake_case__							,			add_bos_token=snake_case__							,			**snake_case__							,			)
			A				=add_bos_token

			with open(snake_case__							,			encoding="utf-8" ) as vocab_handle:
					A				=json.load(snake_case__ )
			A				={v: k for k, v in self.encoder.items()}
			A				=errors  # how to handle errors in decoding
			A				=bytes_to_unicode()
			A				={v: k for k, v in self.byte_encoder.items()}
			with open(snake_case__							,			encoding="utf-8" ) as merges_handle:
					A				=merges_handle.read().split("\n" )[1:-1]
			A				=[tuple(merge.split() ) for merge in bpe_merges]
			A				=dict(zip(snake_case__							,			range(len(snake_case__ ) ) ) )
			A				={}
			A				=add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			A				=re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+" )







	@property
	def        _a					(      self				:			List[Any] ):
			"""simple docstring"""





			return len(self.encoder )







	def        _a					(      self				:			Tuple ):
			"""simple docstring"""





			return dict(self.encoder							,			**self.added_tokens_encoder )







	def        _a					(      self				:			str							,			snake_case__				:			Union[str, Any] ):
			"""simple docstring"""





			if token in self.cache:
					return self.cache[token]
			A				=tuple(snake_case__ )
			A				=get_pairs(snake_case__ )

			if not pairs:
					return token

			while True:
					A				=min(snake_case__							,			key=lambda snake_case__ : self.bpe_ranks.get(snake_case__							,			float("inf" ) ) )
					if bigram not in self.bpe_ranks:
							break
					A			,    A				=bigram
					A				=[]
					A				=0
					while i < len(snake_case__ ):
							try:
									A				=word.index(snake_case__							,			snake_case__ )
							except ValueError:
									new_word.extend(word[i:] )
									break
							else:
									new_word.extend(word[i:j] )
									A				=j

							if word[i] == first and i < len(snake_case__ ) - 1 and word[i + 1] == second:
									new_word.append(first + second )
									i += 2
							else:
									new_word.append(word[i] )
									i += 1
					A				=tuple(snake_case__ )
					A				=new_word
					if len(snake_case__ ) == 1:
							break
					else:
							A				=get_pairs(snake_case__ )
			A				=" ".join(snake_case__ )
			A				=word
			return word







	def        _a					(      self				:			Optional[Any]							,			snake_case__				:			Any							,			snake_case__				:			Optional[Any]=None ):
			"""simple docstring"""





			if self.add_bos_token:
					A				=[self.bos_token_id]
			else:
					A				=[]

			A				=bos_token_ids + token_ids_a

			if token_ids_a is None:
					return output

			return output + bos_token_ids + token_ids_a







	def        _a					(      self				:			List[str]							,			snake_case__				:			List[Any] ):
			"""simple docstring"""





			A				=[]
			for token in re.findall(self.pat							,			snake_case__ ):
					A				="".join(
					    self.byte_encoder[b] for b in token.encode("utf-8" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__ ).split(" " ) )
			return bpe_tokens







	def        _a					(      self				:			Tuple							,			snake_case__				:			List[Any] ):
			"""simple docstring"""





			return self.encoder.get(snake_case__							,			self.encoder.get(self.unk_token ) )







	def        _a					(      self				:			str							,			snake_case__				:			Dict ):
			"""simple docstring"""





			return self.decoder.get(snake_case__ )







	def        _a					(      self				:			Union[str, Any]							,			snake_case__				:			List[Any] ):
			"""simple docstring"""





			A				="".join(snake_case__ )
			A				=bytearray([self.byte_decoder[c] for c in text] ).decode("utf-8"							,			errors=self.errors )
			return text







	def        _a					(      self				:			List[str]							,			snake_case__				:			str							,			snake_case__				:			Optional[str] = None ):
			"""simple docstring"""





			if not os.path.isdir(snake_case__ ):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
					return
			A				=os.path.join(
			    snake_case__							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )
			A				=os.path.join(
			    snake_case__							,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"] )

			with open(snake_case__							,			"w"							,			encoding="utf-8" ) as f:
					f.write(json.dumps(self.encoder							,			indent=2							,			sort_keys=snake_case__							,			ensure_ascii=snake_case__ ) + "\n" )

			A				=0
			with open(snake_case__							,			"w"							,			encoding="utf-8" ) as writer:
					writer.write("#version: 0.2\n" )
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,			key=lambda snake_case__ : kv[1] ):
							if index != token_index:
									logger.warning(
									    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
									    " Please check that the tokenizer is not corrupted!" )
									A				=token_index
							writer.write(" ".join(snake_case__ ) + "\n" )
							index += 1

			return vocab_file, merge_file







	def        _a					(      self				:			str							,			snake_case__				:			Optional[int]							,			snake_case__				:			Optional[int]=False							,			**snake_case__				:			List[Any] ):
			"""simple docstring"""





			A				=kwargs.pop("add_prefix_space"							,			self.add_prefix_space )
			if is_split_into_words or add_prefix_space:
					A				=" " + text
			return (text, kwargs)







	def        _a					(      self				:			str							,			snake_case__				:			Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]							,			snake_case__				:			bool = False							,			snake_case__				:			bool = None							,			snake_case__				:			Optional[List[str]] = None							,			**snake_case__				:			Union[str, Any]							,			):
			"""simple docstring"""





			A				=super()._decode(
			    token_ids=snake_case__							,			skip_special_tokens=snake_case__							,			clean_up_tokenization_spaces=snake_case__							,			**snake_case__							,			)

			if truncate_before_pattern is not None and len(snake_case__ ) > 0:
					A				=self.truncate(snake_case__							,			snake_case__ )

			return decoded_text







	def        _a					(      self				:			Tuple							,			snake_case__				:			List[Any]							,			snake_case__				:			Union[str, Any] ):
			"""simple docstring"""





			def find_re(snake_case__				:			Dict							,			snake_case__				:			Tuple							,			snake_case__				:			int ):
					A				=pattern.search(snake_case__							,			snake_case__ )
					return m.start() if m else -1

			A				=[re.compile(snake_case__							,			re.MULTILINE ) for pattern in truncate_before_pattern]

			A				=list(re.finditer("^print"							,			snake_case__							,			re.MULTILINE ) )

			if len(snake_case__ ) > 1:
					A				=completion[: prints[1].start()]

			A				=list(re.finditer("^def"							,			snake_case__							,			re.MULTILINE ) )

			if len(snake_case__ ) > 1:
					A				=completion[: defs[1].start()]

			A				=0

			A				=[
			    pos for pos in [find_re(snake_case__							,			snake_case__							,			snake_case__ ) for terminal in terminals] if pos != -1
			]

			if len(snake_case__ ) > 0:
					return completion[: min(snake_case__ )]
			else:
					return completion




