




import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel





class     lowerCamelCase__:
      def __init__(				self: Optional[int]    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Any=13    ,       UpperCamelCase_: Optional[int]=7    ,       UpperCamelCase_: Any=True    ,       UpperCamelCase_: Union[str, Any]=True    ,       UpperCamelCase_: Dict=True    ,       UpperCamelCase_: Optional[Any]=True    ,       UpperCamelCase_: List[Any]=99    ,       UpperCamelCase_: Optional[int]=32    ,       UpperCamelCase_: List[Any]=5    ,       UpperCamelCase_: int=4    ,       UpperCamelCase_: List[str]=4    ,       UpperCamelCase_: Union[str, Any]="gelu"    ,       UpperCamelCase_: List[Any]=0.0    ,       UpperCamelCase_: Dict=0.1    ,       UpperCamelCase_: str=True    ,       UpperCamelCase_: Optional[Any]=5_12    ,       UpperCamelCase_: int=16    ,       UpperCamelCase_: Optional[Any]=2    ,       UpperCamelCase_: List[Any]=0.02    ,       UpperCamelCase_: Union[str, Any]=3    ,       UpperCamelCase_: Optional[Any]=4    ,       UpperCamelCase_: Tuple=None    ,       ):
             __lowerCamelCase   =						parent
             __lowerCamelCase   =						batch_size
             __lowerCamelCase   =						seq_length
             __lowerCamelCase   =						is_training
             __lowerCamelCase   =						use_input_mask
             __lowerCamelCase   =						use_token_type_ids
             __lowerCamelCase   =						use_labels
             __lowerCamelCase   =						vocab_size
             __lowerCamelCase   =						hidden_size
             __lowerCamelCase   =						num_hidden_layers
             __lowerCamelCase   =						num_attention_heads
             __lowerCamelCase   =						intermediate_multiple_size
             __lowerCamelCase   =						hidden_act
             __lowerCamelCase   =						hidden_dropout
             __lowerCamelCase   =						attention_dropout
             __lowerCamelCase   =						weight_tying
             __lowerCamelCase   =						max_position_embeddings
             __lowerCamelCase   =						type_vocab_size
             __lowerCamelCase   =						type_sequence_label_size
             __lowerCamelCase   =						initializer_range
             __lowerCamelCase   =						num_labels
             __lowerCamelCase   =						num_choices
             __lowerCamelCase   =						scope
      def 			lowerCAmelCase__						(				self: int   ):
             __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.vocab_size   )

             __lowerCamelCase   =						None
             if self.use_input_mask:
                    __lowerCamelCase   =						random_attention_mask([self.batch_size, self.seq_length]   )

             __lowerCamelCase   =						None
             if self.use_labels:
                    __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.num_labels   )

             __lowerCamelCase   =						self.get_config()

             return config, input_ids, input_mask, token_labels
      def 			lowerCAmelCase__						(				self: Any   ):
             return GPTNeoXJapaneseConfig(
                 vocab_size=self.vocab_size    ,       hidden_size=self.hidden_size    ,       num_hidden_layers=self.num_hidden_layers    ,       num_attention_heads=self.num_attention_heads    ,       intermediate_multiple_size=self.intermediate_multiple_size    ,       hidden_act=self.hidden_act    ,       hidden_dropout=self.hidden_dropout    ,       attention_dropout=self.attention_dropout    ,       weight_tying=self.weight_tying    ,       max_position_embeddings=self.max_position_embeddings    ,       type_vocab_size=self.type_vocab_size    ,       is_decoder=UpperCamelCase_    ,       initializer_range=self.initializer_range    ,       )
      def 			lowerCAmelCase__						(				self: Optional[int]   ):
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						self.prepare_config_and_inputs()

             __lowerCamelCase   =						True

             return config, input_ids, input_mask, token_labels
      def 			lowerCAmelCase__						(				self: Dict    ,       UpperCamelCase_: str    ,       UpperCamelCase_: Optional[int]    ,       UpperCamelCase_: List[Any]   ):
             __lowerCamelCase   =						GPTNeoXJapaneseModel(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             __lowerCamelCase   =						model(UpperCamelCase_    ,       attention_mask=UpperCamelCase_   )
             __lowerCamelCase   =						model(UpperCamelCase_   )
             self.parent.assertEqual(result.last_hidden_state.shape    ,       (self.batch_size, self.seq_length, self.hidden_size)   )
      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: List[Any]    ,       UpperCamelCase_: Union[str, Any]    ,       UpperCamelCase_: List[Any]   ):
             __lowerCamelCase   =						True
             __lowerCamelCase   =						GPTNeoXJapaneseModel(UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             __lowerCamelCase   =						model(UpperCamelCase_    ,       attention_mask=UpperCamelCase_   )
             self.parent.assertEqual(result.last_hidden_state.shape    ,       (self.batch_size, self.seq_length, self.hidden_size)   )
      def 			lowerCAmelCase__						(				self: Union[str, Any]    ,       UpperCamelCase_: Union[str, Any]    ,       UpperCamelCase_: int    ,       UpperCamelCase_: str    ,       UpperCamelCase_: int   ):
             __lowerCamelCase   =						GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()
             __lowerCamelCase   =						model(UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       labels=UpperCamelCase_   )
             self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.seq_length, self.vocab_size)   )
      def 			lowerCAmelCase__						(				self: int    ,       UpperCamelCase_: Union[str, Any]    ,       UpperCamelCase_: List[Any]    ,       UpperCamelCase_: Any   ):
             __lowerCamelCase   =						True
             __lowerCamelCase   =						GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_   )
             model.to(UpperCamelCase_   )
             model.eval()

             # first forward pass
             __lowerCamelCase   =						model(UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       use_cache=UpperCamelCase_   )
             __lowerCamelCase   =						outputs.past_key_values

             # create hypothetical multiple next token and extent to next_input_ids
             __lowerCamelCase   =						ids_tensor((self.batch_size, 3)    ,       config.vocab_size   )
             __lowerCamelCase   =						ids_tensor((self.batch_size, 3)    ,       vocab_size=2   )

             # append to next input_ids and
             __lowerCamelCase   =						torch.cat([input_ids, next_tokens]    ,       dim=-1   )
             __lowerCamelCase   =						torch.cat([input_mask, next_mask]    ,       dim=-1   )

             __lowerCamelCase   =						model(UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       output_hidden_states=UpperCamelCase_   )
             __lowerCamelCase   =						output_from_no_past["""hidden_states"""][0]
             __lowerCamelCase   =						model(
                 UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       past_key_values=UpperCamelCase_    ,       output_hidden_states=UpperCamelCase_    ,       )["""hidden_states"""][0]

             # select random slice
             __lowerCamelCase   =						ids_tensor((1,)    ,       output_from_past.shape[-1]   ).item()
             __lowerCamelCase   =						output_from_no_past[:, -3:, random_slice_idx].detach()
             __lowerCamelCase   =						output_from_past[:, :, random_slice_idx].detach()

             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(UpperCamelCase_    ,       UpperCamelCase_    ,       atol=1E-3   )   )




      def 			lowerCAmelCase__						(				self: Any   ):
             __lowerCamelCase   =						self.prepare_config_and_inputs()
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						config_and_inputs
             __lowerCamelCase   =						{"""input_ids""": input_ids, """attention_mask""": input_mask}
             return config, inputs_dict





@require_torch
class     lowerCamelCase__(					__lowerCamelCase ,     __lowerCamelCase ,     unittest.TestCase):
      UpperCAmelCase__      :	List[Any]    		=   (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
      UpperCAmelCase__      :	Dict    		=   (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
      UpperCAmelCase__      :	Dict    		=   (
          {'feature-extraction': GPTNeoXJapaneseModel, 'text-generation': GPTNeoXJapaneseForCausalLM}
          if is_torch_available()
          else {}
      )
      UpperCAmelCase__      :	Optional[int]    		=   False
      UpperCAmelCase__      :	List[Any]    		=   False
      UpperCAmelCase__      :	Tuple    		=   False
      UpperCAmelCase__      :	Optional[int]    		=   False
      def 			lowerCAmelCase__						(				self: int   ):
             __lowerCamelCase   =						GPTNeoXJapaneseModelTester(self   )
             __lowerCamelCase   =						ConfigTester(self    ,       config_class=UpperCamelCase_    ,       hidden_size=37   )
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             self.config_tester.run_common_tests()
      def 			lowerCAmelCase__						(				self: int   ):
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: str   ):
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs_for_decoder()
             self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Any   ):
             # This regression test was failing with PyTorch < 1.3
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs_for_decoder()

             __lowerCamelCase   =						None

             self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: List[str]   ):
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_decoder_model_past_large_inputs(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: str   ):
             __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_causal_lm(*UpperCamelCase_   )




      @slow
      def 			lowerCAmelCase__						(				self: Dict   ):
             __lowerCamelCase   =						"""abeja/gpt-neox-japanese-2.7b"""

             __lowerCamelCase   =						["""データサイエンティストとは、""", """100年後に必要とされる会社は、""", """フルリモートの環境で働くために必要なことは、""", """国境の長いトンネルを抜けると""", """美味しい日本食といえば、"""]

             __lowerCamelCase   =						[
                 """データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。""",
                 """100年後に必要とされる会社は、「人」が中心の会社です。""",
                 """フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。""",
                 """国境の長いトンネルを抜けると、そこは雪国だった。""",
                 """美味しい日本食といえば、やっぱりお寿司ですよね。""",
             ]

             __lowerCamelCase   =						GPTNeoXJapaneseTokenizer.from_pretrained(UpperCamelCase_   )
             __lowerCamelCase   =						GPTNeoXJapaneseForCausalLM.from_pretrained(UpperCamelCase_   )

             __lowerCamelCase   =						[]
             for prompt in prompts:
                    __lowerCamelCase   =						tokenizer(UpperCamelCase_    ,       return_tensors="""pt"""   ).input_ids
                    __lowerCamelCase   =						model.generate(UpperCamelCase_    ,       max_length=50   )
                    __lowerCamelCase   =						tokenizer.batch_decode(UpperCamelCase_    ,       skip_special_tokens=UpperCamelCase_   )
                    predicted_outputs += generated_string
             self.assertListEqual(UpperCamelCase_    ,       UpperCamelCase_   )







import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    FEATURE_EXTRACTOR_MAPPING,
    AutoConfig,
    AutoFeatureExtractor,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


UpperCAmelCase_									=	get_tests_dir('fixtures')
UpperCAmelCase_									=	get_tests_dir('fixtures/dummy_feature_extractor_config.json')
UpperCAmelCase_									=	get_tests_dir('fixtures/dummy-config.json')





class     lowerCamelCase__(					unittest.TestCase):
      def 			lowerCAmelCase__						(				self: List[str]   ):
             __lowerCamelCase   =						0
      def 			lowerCAmelCase__						(				self: List[str]   ):
             __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained("""facebook/wav2vec2-base-960h"""   )
             self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Union[str, Any]   ):
             __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_   )
             self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: int   ):
             with tempfile.TemporaryDirectory() as tmpdirname:
                    __lowerCamelCase   =						WavaVecaConfig()

                    # remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_   ).to_dict()

                    config_dict.pop("""feature_extractor_type"""   )
                    __lowerCamelCase   =						WavaVecaFeatureExtractor(**UpperCamelCase_   )

                    # save in new folder
                    model_config.save_pretrained(UpperCamelCase_   )
                    config.save_pretrained(UpperCamelCase_   )

                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_   )

                    # make sure private variable is not incorrectly saved
                    __lowerCamelCase   =						json.loads(config.to_json_string()   )
                    self.assertTrue("""_processor_class""" not in dict_as_saved   )

             self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Tuple   ):
             __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_   )
             self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: int   ):
             with self.assertRaisesRegex(
                 UpperCamelCase_    ,       """bert-base is not a local folder and is not a valid model identifier"""   ):
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained("""bert-base"""   )
      def 			lowerCAmelCase__						(				self: Tuple   ):
             with self.assertRaisesRegex(
                 UpperCamelCase_    ,       r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""   ):
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_    ,       revision="""aaaaaa"""   )
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             with self.assertRaisesRegex(
                 UpperCamelCase_    ,       """hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json."""    ,       ):
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained("""hf-internal-testing/config-no-model"""   )
      def 			lowerCAmelCase__						(				self: Tuple   ):
             # If remote code is not set, we will time out when asking whether to load the model.
             with self.assertRaises(UpperCamelCase_   ):
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                        """hf-internal-testing/test_dynamic_feature_extractor"""   )
             # If remote code is disabled, we can't load this config.
             with self.assertRaises(UpperCamelCase_   ):
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                        """hf-internal-testing/test_dynamic_feature_extractor"""    ,       trust_remote_code=UpperCamelCase_   )

             __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                 """hf-internal-testing/test_dynamic_feature_extractor"""    ,       trust_remote_code=UpperCamelCase_   )
             self.assertEqual(feature_extractor.__class__.__name__    ,       """NewFeatureExtractor"""   )

             # Test feature extractor can be reloaded.
             with tempfile.TemporaryDirectory() as tmp_dir:
                    feature_extractor.save_pretrained(UpperCamelCase_   )
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_    ,       trust_remote_code=UpperCamelCase_   )
             self.assertEqual(reloaded_feature_extractor.__class__.__name__    ,       """NewFeatureExtractor"""   )
      def 			lowerCAmelCase__						(				self: Any   ):
             try:
                    AutoConfig.register("""custom"""    ,       UpperCamelCase_   )
                    AutoFeatureExtractor.register(UpperCamelCase_    ,       UpperCamelCase_   )
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(UpperCamelCase_   ):
                           AutoFeatureExtractor.register(UpperCamelCase_    ,       UpperCamelCase_   )

                    # Now that the config is registered, it can be used as any other config with the auto-API
                    __lowerCamelCase   =						CustomFeatureExtractor.from_pretrained(UpperCamelCase_   )
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           feature_extractor.save_pretrained(UpperCamelCase_   )
                           __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(UpperCamelCase_   )
                           self.assertIsInstance(UpperCamelCase_    ,       UpperCamelCase_   )

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]




      def 			lowerCAmelCase__						(				self: Dict   ):





             class     lowerCamelCase__(					__lowerCamelCase):
                   UpperCAmelCase__      :	str    		=   True

             try:
                    AutoConfig.register("""custom"""    ,       UpperCamelCase_   )
                    AutoFeatureExtractor.register(UpperCamelCase_    ,       UpperCamelCase_   )
                    # If remote code is not set, the default is to use local
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                        """hf-internal-testing/test_dynamic_feature_extractor"""   )
                    self.assertEqual(feature_extractor.__class__.__name__    ,       """NewFeatureExtractor"""   )
                    self.assertTrue(feature_extractor.is_local   )

                    # If remote code is disabled, we load the local one.
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                        """hf-internal-testing/test_dynamic_feature_extractor"""    ,       trust_remote_code=UpperCamelCase_   )
                    self.assertEqual(feature_extractor.__class__.__name__    ,       """NewFeatureExtractor"""   )
                    self.assertTrue(feature_extractor.is_local   )

                    # If remote is enabled, we load from the Hub
                    __lowerCamelCase   =						AutoFeatureExtractor.from_pretrained(
                        """hf-internal-testing/test_dynamic_feature_extractor"""    ,       trust_remote_code=UpperCamelCase_   )
                    self.assertEqual(feature_extractor.__class__.__name__    ,       """NewFeatureExtractor"""   )
                    self.assertTrue(not hasattr(UpperCamelCase_    ,       """is_local"""   )   )

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
                           del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]

