




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__       :      List[str]  =   '▁'

SCREAMING_SNAKE_CASE__       :      Any  =   {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE__       :      Dict  =   {
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    }
}

SCREAMING_SNAKE_CASE__       :      List[Any]  =   {
    'google/reformer-crime-and-punishment': 524288,
}




class 		UpperCamelCase__	(lowerCAmelCase__     ):

    '''simple docstring'''



    lowerCamelCase_      :	Any	     =      VOCAB_FILES_NAMES
    lowerCamelCase_      :	List[Any]	     =      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_      :	str	     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_      :	Optional[Any]	     =      ["""input_ids""", """attention_mask"""]







    def __init__(    self     ,  UpperCamelCase__     ,  UpperCamelCase__="</s>"     ,  UpperCamelCase__="<unk>"     ,  UpperCamelCase__=[]     ,  UpperCamelCase__ = None     ,  **UpperCamelCase__     ,  )     ->      None:
           lowerCamelCase   :      List[Any]			 =					{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               eos_token=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  additional_special_tokens=UpperCamelCase__     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase__     ,  )

           lowerCamelCase   :      str			 =					vocab_file
           lowerCamelCase   :      List[Any]			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(UpperCamelCase__     )







    @property
    def     _lowercase     (    self     )     ->      Dict:
           return self.sp_model.get_piece_size()







    def     _lowercase     (    self     )     ->      Dict[str, int]:
           lowerCamelCase   :      Optional[Any]			 =					{self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
           vocab.update(self.added_tokens_encoder     )
           return vocab







    def __getstate__(    self     )     ->      str:
           lowerCamelCase   :      Optional[int]			 =					self.__dict__.copy()
           lowerCamelCase   :      Dict			 =					None
           return state







    def __setstate__(    self     ,  UpperCamelCase__     )     ->      Union[str, Any]:
           lowerCamelCase   :      Any			 =					d

           # for backward compatibility
           if not hasattr(self     ,  "sp_model_kwargs"     ):
                  lowerCamelCase   :      Any			 =					{}

           lowerCamelCase   :      int			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.vocab_file     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      List[str]:
           return self.sp_model.encode(UpperCamelCase__     ,  out_type=UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Union[str, Any]:
           return self.sp_model.piece_to_id(UpperCamelCase__     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      int:
           if index < self.sp_model.get_piece_size():
                  lowerCamelCase   :      Dict			 =					self.sp_model.IdToPiece(UpperCamelCase__     )
           return token







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Dict:
           lowerCamelCase   :      Any			 =					[]
           lowerCamelCase   :      str			 =					""
           for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                         out_string += self.sp_model.decode(UpperCamelCase__     ) + token
                         lowerCamelCase   :      Union[str, Any]			 =					[]
                  else:
                         current_sub_tokens.append(UpperCamelCase__     )
           out_string += self.sp_model.decode(UpperCamelCase__     )
           return out_string.strip()







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      Tuple[str]:
           if not os.path.isdir(UpperCamelCase__     ):
                  logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
                  return
           lowerCamelCase   :      Optional[int]			 =					os.path.join(
               UpperCamelCase__     ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

           if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
                  copyfile(self.vocab_file     ,  UpperCamelCase__     )
           elif not os.path.isfile(self.vocab_file     ):
                  with open(UpperCamelCase__     ,  "wb"     ) as fi:
                         lowerCamelCase   :      Tuple			 =					self.sp_model.serialized_model_proto()
                         fi.write(UpperCamelCase__     )

           return (out_vocab_file,)










import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
       OpenAIGPTConfig,
       OpenAIGPTDoubleHeadsModel,
       OpenAIGPTForSequenceClassification,
       OpenAIGPTLMHeadModel,
       OpenAIGPTModel,
   )
class  SCREAMING_SNAKE_CASE__  :
      def __init__(       self				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Tuple=1_3     ,							SCREAMING_SNAKE_CASE__				:       str=7     ,							SCREAMING_SNAKE_CASE__				:       List[str]=True     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=9_9     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=3_2     ,							SCREAMING_SNAKE_CASE__				:       List[str]=5     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=4     ,							SCREAMING_SNAKE_CASE__				:       Tuple=3_7     ,							SCREAMING_SNAKE_CASE__				:       Any="gelu"     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.1     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.1     ,							SCREAMING_SNAKE_CASE__				:       int=5_1_2     ,							SCREAMING_SNAKE_CASE__				:       int=1_6     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=2     ,							SCREAMING_SNAKE_CASE__				:       Any=0.02     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=3     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=4     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=None     ,							)		->			Any:
             a_					:		Tuple							=  parent
             a_					:		int							=  batch_size
             a_					:		Tuple							=  seq_length
             a_					:		List[Any]							=  is_training
             a_					:		List[str]							=  use_token_type_ids
             a_					:		Dict							=  use_labels
             a_					:		Any							=  vocab_size
             a_					:		List[str]							=  hidden_size
             a_					:		Tuple							=  num_hidden_layers
             a_					:		List[Any]							=  num_attention_heads
             a_					:		Dict							=  intermediate_size
             a_					:		Any							=  hidden_act
             a_					:		List[str]							=  hidden_dropout_prob
             a_					:		Tuple							=  attention_probs_dropout_prob
             a_					:		Optional[Any]							=  max_position_embeddings
             a_					:		List[Any]							=  type_vocab_size
             a_					:		int							=  type_sequence_label_size
             a_					:		List[Any]							=  initializer_range
             a_					:		List[str]							=  num_labels
             a_					:		Union[str, Any]							=  num_choices
             a_					:		str							=  scope
             a_					:		Tuple							=  self.vocab_size - 1
      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     )		->			Any:
             a_					:		Optional[int]							=  ids_tensor([self.batch_size, self.seq_length]     ,							self.vocab_size     )

             a_					:		Any							=  None
             if self.use_token_type_ids:
                    a_					:		Tuple							=  ids_tensor([self.batch_size, self.seq_length]     ,							self.type_vocab_size     )

             a_					:		List[Any]							=  None
             a_					:		Union[str, Any]							=  None
             a_					:		List[Any]							=  None
             if self.use_labels:
                    a_					:		Tuple							=  ids_tensor([self.batch_size]     ,							self.type_sequence_label_size     )
                    a_					:		Any							=  ids_tensor([self.batch_size, self.seq_length]     ,							self.num_labels     )
                    a_					:		List[Any]							=  ids_tensor([self.batch_size]     ,							self.num_choices     )

             a_					:		Union[str, Any]							=  OpenAIGPTConfig(
                 vocab_size=self.vocab_size     ,							n_embd=self.hidden_size     ,							n_layer=self.num_hidden_layers     ,							n_head=self.num_attention_heads     ,							n_positions=self.max_position_embeddings     ,							pad_token_id=self.pad_token_id     ,							)

             a_					:		List[str]							=  ids_tensor([self.num_hidden_layers, self.num_attention_heads]     ,							2     )

             return (
                 config,
                 input_ids,
                 head_mask,
                 token_type_ids,
                 sequence_labels,
                 token_labels,
                 choice_labels,
             )
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       Any     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     ,							SCREAMING_SNAKE_CASE__				:       Any     ,							SCREAMING_SNAKE_CASE__				:       List[str]     ,							*SCREAMING_SNAKE_CASE__				:       Tuple     )		->			Union[str, Any]:
             a_					:		Dict							=  OpenAIGPTModel(config=SCREAMING_SNAKE_CASE__     )
             model.to(SCREAMING_SNAKE_CASE__     )
             model.eval()

             a_					:		str							=  model(SCREAMING_SNAKE_CASE__     ,							token_type_ids=SCREAMING_SNAKE_CASE__     ,							head_mask=SCREAMING_SNAKE_CASE__     )
             a_					:		Dict							=  model(SCREAMING_SNAKE_CASE__     ,							token_type_ids=SCREAMING_SNAKE_CASE__     )
             a_					:		Dict							=  model(SCREAMING_SNAKE_CASE__     )

             self.parent.assertEqual(result.last_hidden_state.shape     ,							(self.batch_size, self.seq_length, self.hidden_size)     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     ,							*SCREAMING_SNAKE_CASE__				:       Optional[int]     )		->			Any:
             a_					:		str							=  OpenAIGPTLMHeadModel(SCREAMING_SNAKE_CASE__     )
             model.to(SCREAMING_SNAKE_CASE__     )
             model.eval()

             a_					:		Optional[int]							=  model(SCREAMING_SNAKE_CASE__     ,							token_type_ids=SCREAMING_SNAKE_CASE__     ,							labels=SCREAMING_SNAKE_CASE__     )
             self.parent.assertEqual(result.loss.shape     ,							()     )
             self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.seq_length, self.vocab_size)     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[Any]     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]     ,							SCREAMING_SNAKE_CASE__				:       int     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]     ,							*SCREAMING_SNAKE_CASE__				:       Union[str, Any]     )		->			Dict:
             a_					:		int							=  OpenAIGPTDoubleHeadsModel(SCREAMING_SNAKE_CASE__     )
             model.to(SCREAMING_SNAKE_CASE__     )
             model.eval()

             a_					:		str							=  model(SCREAMING_SNAKE_CASE__     ,							token_type_ids=SCREAMING_SNAKE_CASE__     ,							labels=SCREAMING_SNAKE_CASE__     )
             self.parent.assertEqual(result.loss.shape     ,							()     )
             self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.seq_length, self.vocab_size)     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       List[Any]     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     ,							*SCREAMING_SNAKE_CASE__				:       str     )		->			List[str]:
             a_					:		Any							=  self.num_labels
             a_					:		Dict							=  OpenAIGPTForSequenceClassification(SCREAMING_SNAKE_CASE__     )
             model.to(SCREAMING_SNAKE_CASE__     )
             model.eval()

             a_					:		Dict							=  ids_tensor([self.batch_size]     ,							self.type_sequence_label_size     )
             a_					:		Any							=  model(SCREAMING_SNAKE_CASE__     ,							token_type_ids=SCREAMING_SNAKE_CASE__     ,							labels=SCREAMING_SNAKE_CASE__     )
             self.parent.assertEqual(result.logits.shape     ,							(self.batch_size, self.num_labels)     )

      def        SCREAMING_SNAKE_CASE							(       self				:       Union[str, Any]     )		->			Tuple:
             a_					:		Optional[Any]							=  self.prepare_config_and_inputs()
             (
                 (
                 a_
             )							,		(
                 a_
             )							,		(
                 a_
             )							,		(
                 a_
             )							,		(
                 a_
             )							,		(
                 a_
             )							,		(
                 a_
             )							,		
             )					:		Optional[Any]							=  config_and_inputs
             a_					:		Optional[int]							=  {
                 'input_ids': input_ids,
                 'token_type_ids': token_type_ids,
                 'head_mask': head_mask,
             }

             return config, inputs_dict
@require_torch
class  SCREAMING_SNAKE_CASE__  (				lowercase__						,    lowercase__						,    lowercase__						,    unittest.TestCase			):
      snake_case__				:     Tuple        =						(
          (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
          if is_torch_available()
          else ()
      )
      snake_case__				:     List[str]        =						(
          (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
      )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
      snake_case__				:     Dict        =						(
          {
              '''feature-extraction''': OpenAIGPTModel,
              '''text-classification''': OpenAIGPTForSequenceClassification,
              '''text-generation''': OpenAIGPTLMHeadModel,
              '''zero-shot''': OpenAIGPTForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]     ,							SCREAMING_SNAKE_CASE__				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       List[str]     )		->			Dict:
             if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                    # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                    # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
                    # tiny config could not be created.
                    return True

             return False
      def        SCREAMING_SNAKE_CASE							(       self				:       int     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Dict     ,							SCREAMING_SNAKE_CASE__				:       Any=False     )		->			List[str]:
             a_					:		str							=  super()._prepare_for_class(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     ,							return_labels=SCREAMING_SNAKE_CASE__     )

             if return_labels:
                    if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                           a_					:		Optional[Any]							=  torch.zeros(
                               (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)     ,							dtype=torch.long     ,							device=SCREAMING_SNAKE_CASE__     ,							)
                           a_					:		str							=  inputs_dict['labels']
                           a_					:		Optional[int]							=  inputs_dict['labels']
                           a_					:		Optional[int]							=  torch.zeros(
                               (self.model_tester.batch_size, self.model_tester.num_choices)     ,							dtype=torch.long     ,							device=SCREAMING_SNAKE_CASE__     ,							)
                           a_					:		Union[str, Any]							=  torch.zeros(
                               self.model_tester.batch_size     ,							dtype=torch.long     ,							device=SCREAMING_SNAKE_CASE__     )
             return inputs_dict
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			List[Any]:
             a_					:		str							=  OpenAIGPTModelTester(self     )
             a_					:		int							=  ConfigTester(self     ,							config_class=SCREAMING_SNAKE_CASE__     ,							n_embd=3_7     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     )		->			Tuple:
             self.config_tester.run_common_tests()
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     )		->			Tuple:
             a_					:		Dict							=  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_model(*SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[int]     )		->			Tuple:
             a_					:		Any							=  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_lm_head_model(*SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Any     )		->			Optional[Any]:
             a_					:		Optional[int]							=  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_double_lm_head_model(*SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[Any]     )		->			Optional[Any]:
             a_					:		int							=  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*SCREAMING_SNAKE_CASE__     )

      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     )		->			str:
             for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    a_					:		str							=  OpenAIGPTModel.from_pretrained(SCREAMING_SNAKE_CASE__     )
                    self.assertIsNotNone(SCREAMING_SNAKE_CASE__     )
@require_torch
class  SCREAMING_SNAKE_CASE__  (				unittest.TestCase			):
      @slow
      def        SCREAMING_SNAKE_CASE							(       self				:       Dict     )		->			int:
             a_					:		Dict							=  OpenAIGPTLMHeadModel.from_pretrained('openai-gpt'     )
             model.to(SCREAMING_SNAKE_CASE__     )
             a_					:		List[Any]							=  torch.tensor([[4_8_1, 4_7_3_5, 5_4_4]]     ,							dtype=torch.long     ,							device=SCREAMING_SNAKE_CASE__     )  # the president is
             a_					:		Tuple							=  [
                 4_8_1,
                 4_7_3_5,
                 5_4_4,
                 2_4_6,
                 9_6_3,
                 8_7_0,
                 7_6_2,
                 2_3_9,
                 2_4_4,
                 4_0_4_7_7,
                 2_4_4,
                 2_4_9,
                 7_1_9,
                 8_8_1,
                 4_8_7,
                 5_4_4,
                 2_4_0,
                 2_4_4,
                 6_0_3,
                 4_8_1,
             ]  # the president is a very good man. " \n " i\'m sure he is, " said the

             a_					:		Dict							=  model.generate(SCREAMING_SNAKE_CASE__     ,							do_sample=SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(output_ids[0].tolist()     ,							SCREAMING_SNAKE_CASE__     )


