






import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
         CTRLForSequenceClassification,
         CTRLLMHeadModel,
         CTRLModel,
     )






class 		lowercase_					:






      '''simple docstring'''




      def __init__(      self      :						List[Any] ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						Union[str, Any]=14 ,					__UpperCAmelCase      :						Any=7 ,					__UpperCAmelCase      :						Tuple=True ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						Any=True ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						int=99 ,					__UpperCAmelCase      :						Tuple=32 ,					__UpperCAmelCase      :						int=5 ,					__UpperCAmelCase      :						str=4 ,					__UpperCAmelCase      :						Optional[int]=37 ,					__UpperCAmelCase      :						Tuple="gelu" ,					__UpperCAmelCase      :						Optional[int]=0.1 ,					__UpperCAmelCase      :						Tuple=0.1 ,					__UpperCAmelCase      :						str=512 ,					__UpperCAmelCase      :						int=16 ,					__UpperCAmelCase      :						Dict=2 ,					__UpperCAmelCase      :						Any=0.02 ,					__UpperCAmelCase      :						Tuple=3 ,					__UpperCAmelCase      :						Tuple=4 ,					__UpperCAmelCase      :						Optional[int]=None ,					)    ->Union[str, Any]:




       """simple docstring"""


       a		  =      parent
       a		  =      batch_size
       a		  =      seq_length
       a		  =      is_training
       a		  =      use_token_type_ids
       a		  =      use_input_mask
       a		  =      use_labels
       a		  =      use_mc_token_ids
       a		  =      vocab_size
       a		  =      hidden_size
       a		  =      num_hidden_layers
       a		  =      num_attention_heads
       a		  =      intermediate_size
       a		  =      hidden_act
       a		  =      hidden_dropout_prob
       a		  =      attention_probs_dropout_prob
       a		  =      max_position_embeddings
       a		  =      type_vocab_size
       a		  =      type_sequence_label_size
       a		  =      initializer_range
       a		  =      num_labels
       a		  =      num_choices
       a		  =      scope
       a		  =      self.vocab_size - 1







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Optional[Any]:




       """simple docstring"""


       a		  =      ids_tensor([self.batch_size, self.seq_length] ,					self.vocab_size  )

       a		  =      None
       if self.use_input_mask:
        a		  =      random_attention_mask([self.batch_size, self.seq_length]  )

       a		  =      None
       if self.use_token_type_ids:
        a		  =      ids_tensor([self.batch_size, self.seq_length] ,					self.type_vocab_size  )

       a		  =      None
       if self.use_mc_token_ids:
        a		  =      ids_tensor([self.batch_size, self.num_choices] ,					self.seq_length  )

       a		  =      None
       a		  =      None
       a		  =      None
       if self.use_labels:
        a		  =      ids_tensor([self.batch_size] ,					self.type_sequence_label_size  )
        a		  =      ids_tensor([self.batch_size, self.seq_length] ,					self.num_labels  )
        a		  =      ids_tensor([self.batch_size] ,					self.num_choices  )

       a		  =      self.get_config()

       a		  =      ids_tensor([self.num_hidden_layers, self.num_attention_heads] ,					2  )

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Union[str, Any]:




       """simple docstring"""


       return CTRLConfig(
           vocab_size=self.vocab_size ,					n_embd=self.hidden_size ,					n_layer=self.num_hidden_layers ,					n_head=self.num_attention_heads ,					n_positions=self.max_position_embeddings ,					pad_token_id=self.pad_token_id ,					)







      def       __lowerCAmelCase					(      self      :						Optional[Any] ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						Any ,					__UpperCAmelCase      :						List[str] ,					__UpperCAmelCase      :						Any ,					__UpperCAmelCase      :						List[Any] ,					*__UpperCAmelCase      :						List[str]  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      CTRLModel(config=__UpperCAmelCase  )
       model.to(__UpperCAmelCase  )
       model.eval()

       model(__UpperCAmelCase ,					token_type_ids=__UpperCAmelCase ,					head_mask=__UpperCAmelCase  )
       model(__UpperCAmelCase ,					token_type_ids=__UpperCAmelCase  )
       a		  =      model(__UpperCAmelCase  )
       self.parent.assertEqual(result.last_hidden_state.shape ,					(self.batch_size, self.seq_length, self.hidden_size)  )
       self.parent.assertEqual(len(result.past_key_values  ) ,					config.n_layer  )







      def       __lowerCAmelCase					(      self      :						str ,					__UpperCAmelCase      :						Dict ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						List[str] ,					__UpperCAmelCase      :						List[str] ,					*__UpperCAmelCase      :						Tuple  )    ->Optional[int]:




       """simple docstring"""


       a		  =      CTRLLMHeadModel(__UpperCAmelCase  )
       model.to(__UpperCAmelCase  )
       model.eval()

       a		  =      model(__UpperCAmelCase ,					token_type_ids=__UpperCAmelCase ,					labels=__UpperCAmelCase  )
       self.parent.assertEqual(result.loss.shape ,					()  )
       self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.seq_length, self.vocab_size)  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->List[Any]:




       """simple docstring"""


       a		  =      self.prepare_config_and_inputs()

       (
           (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     (
           a
       )  ,     
       )		  =      config_and_inputs

       a		  =      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''head_mask''': head_mask}

       return config, inputs_dict







      def       __lowerCAmelCase					(      self      :						Any ,					__UpperCAmelCase      :						int ,					__UpperCAmelCase      :						Any ,					__UpperCAmelCase      :						Optional[Any] ,					__UpperCAmelCase      :						Any ,					*__UpperCAmelCase      :						Dict  )    ->Optional[int]:




       """simple docstring"""


       a		  =      self.num_labels
       a		  =      CTRLForSequenceClassification(__UpperCAmelCase  )
       model.to(__UpperCAmelCase  )
       model.eval()
       a		  =      ids_tensor([self.batch_size] ,					self.type_sequence_label_size  )
       a		  =      model(__UpperCAmelCase ,					token_type_ids=__UpperCAmelCase ,					labels=__UpperCAmelCase  )
       self.parent.assertEqual(result.logits.shape ,					(self.batch_size, self.num_labels)  )






@require_torch
class 		lowercase_					(	lowercase ,  lowercase ,  lowercase ,  unittest.TestCase   ):






      '''simple docstring'''




      __snake_case       =  (CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
      __snake_case       =  (CTRLLMHeadModel,) if is_torch_available() else ()
      __snake_case       =  (
          {
              '''feature-extraction''': CTRLModel,
              '''text-classification''': CTRLForSequenceClassification,
              '''text-generation''': CTRLLMHeadModel,
              '''zero-shot''': CTRLForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      __snake_case       =  True
      __snake_case       =  False
      __snake_case       =  False







      def       __lowerCAmelCase					(      self      :						Dict ,					__UpperCAmelCase      :						Dict ,					__UpperCAmelCase      :						Union[str, Any] ,					__UpperCAmelCase      :						Dict ,					__UpperCAmelCase      :						Dict ,					__UpperCAmelCase      :						Union[str, Any]  )    ->Any:




       """simple docstring"""


       if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
        # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
        # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
        # config could not be created.
        return True

       return False







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->str:




       """simple docstring"""


       a		  =      CTRLModelTester(self  )
       a		  =      ConfigTester(self ,					config_class=__UpperCAmelCase ,					n_embd=37  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->List[str]:




       """simple docstring"""


       super().tearDown()
       # clean-up as much as possible GPU memory occupied by PyTorch
       gc.collect()
       torch.cuda.empty_cache()







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->Dict:




       """simple docstring"""


       self.config_tester.run_common_tests()







      def       __lowerCAmelCase					(      self      :						Dict  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_ctrl_model(*__UpperCAmelCase  )







      def       __lowerCAmelCase					(      self      :						List[str]  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_lm_head_model(*__UpperCAmelCase  )







      @unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''  )
      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Any:




       """simple docstring"""


       pass







      @slow
      def       __lowerCAmelCase					(      self      :						Any  )    ->List[Any]:




       """simple docstring"""


       for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        a		  =      CTRLModel.from_pretrained(__UpperCAmelCase  )
        self.assertIsNotNone(__UpperCAmelCase  )







      @unittest.skip('''The model doesn\'t support left padding'''  )  # and it's not used enough to be worth fixing :)
      def       __lowerCAmelCase					(      self      :						Dict  )    ->str:




       """simple docstring"""


       pass






@require_torch
class 		lowercase_					(	unittest.TestCase   ):






      '''simple docstring'''




      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->int:




       """simple docstring"""


       super().tearDown()
       # clean-up as much as possible GPU memory occupied by PyTorch
       gc.collect()
       torch.cuda.empty_cache()







      @slow
      def       __lowerCAmelCase					(      self      :						str  )    ->Union[str, Any]:




       """simple docstring"""


       a		  =      CTRLLMHeadModel.from_pretrained('''ctrl'''  )
       model.to(__UpperCAmelCase  )
       a		  =      torch.tensor(
           [[11_859, 0, 1_611, 8]] ,					dtype=torch.long ,					device=__UpperCAmelCase  )  # Legal the president is
       a		  =      [
           11_859,
           0,
           1_611,
           8,
           5,
           150,
           26_449,
           2,
           19,
           348,
           469,
           3,
           2_595,
           48,
           20_740,
           246_533,
           246_533,
           19,
           30,
           5,
       ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

       a		  =      model.generate(__UpperCAmelCase ,					do_sample=__UpperCAmelCase  )
       self.assertListEqual(output_ids[0].tolist() ,					__UpperCAmelCase  )










import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 		lowercase_					(	lowercase ,  unittest.TestCase   ):






      '''simple docstring'''




      __snake_case       =  ProphetNetTokenizer
      __snake_case       =  False







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Optional[Any]:




       """simple docstring"""


       super().setUp()

       a		  =      [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       a		  =      os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']  )
       with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''  ) as vocab_writer:
        vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )







      def       __lowerCAmelCase					(      self      :						List[str] ,					__UpperCAmelCase      :						str  )    ->Dict:




       """simple docstring"""


       a		  =      '''UNwant\u00E9d,running'''
       a		  =      '''unwanted, running'''
       return input_text, output_text







      def       __lowerCAmelCase					(      self      :						Optional[int]  )    ->Optional[Any]:




       """simple docstring"""


       a		  =      self.tokenizer_class(self.vocab_file  )

       a		  =      tokenizer.tokenize('''UNwant\u00E9d,running'''  )
       self.assertListEqual(__UpperCAmelCase ,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase  ) ,					[9, 6, 7, 12, 10, 11]  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Any:




       """simple docstring"""


       a		  =      BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  ) ,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->int:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ) ,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->Optional[int]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''h\u00E9llo''']  )







      def       __lowerCAmelCase					(      self      :						Dict  )    ->str:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->Dict:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  ) ,					['''hello''']  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Optional[Any]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Tuple:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						int  )    ->Optional[int]:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  ) ,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->int:




       """simple docstring"""


       a		  =      BasicTokenizer(do_lower_case=__UpperCAmelCase ,					never_split=['''[UNK]''']  )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )







      def       __lowerCAmelCase					(      self      :						Union[str, Any]  )    ->int:




       """simple docstring"""


       a		  =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       a		  =      {}
       for i, token in enumerate(__UpperCAmelCase  ):
        a		  =      i
       a		  =      WordpieceTokenizer(vocab=__UpperCAmelCase ,					unk_token='''[UNK]'''  )

       self.assertListEqual(tokenizer.tokenize(''''''  ) ,					[]  )

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''  ) ,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  ) ,					['''[UNK]''', '''runn''', '''##ing''']  )







      @require_torch
      def       __lowerCAmelCase					(      self      :						int  )    ->int:




       """simple docstring"""


       a		  =      self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

       a		  =      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
       a		  =      [1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
       a		  =      tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					return_tensors='''pt'''  )
       self.assertIsInstance(__UpperCAmelCase ,					__UpperCAmelCase  )
       a		  =      list(batch.input_ids.numpy()[0]  )
       self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase  )

       self.assertEqual((2, 9) ,					batch.input_ids.shape  )
       self.assertEqual((2, 9) ,					batch.attention_mask.shape  )







      def       __lowerCAmelCase					(      self      :						Optional[Any]  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_whitespace(''' '''  )  )
       self.assertTrue(_is_whitespace('''\t'''  )  )
       self.assertTrue(_is_whitespace('''\r'''  )  )
       self.assertTrue(_is_whitespace('''\n'''  )  )
       self.assertTrue(_is_whitespace('''\u00A0'''  )  )

       self.assertFalse(_is_whitespace('''A'''  )  )
       self.assertFalse(_is_whitespace('''-'''  )  )







      def       __lowerCAmelCase					(      self      :						Any  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_control('''\u0005'''  )  )

       self.assertFalse(_is_control('''A'''  )  )
       self.assertFalse(_is_control(''' '''  )  )
       self.assertFalse(_is_control('''\t'''  )  )
       self.assertFalse(_is_control('''\r'''  )  )







      def       __lowerCAmelCase					(      self      :						List[Any]  )    ->List[str]:




       """simple docstring"""


       self.assertTrue(_is_punctuation('''-'''  )  )
       self.assertTrue(_is_punctuation('''$'''  )  )
       self.assertTrue(_is_punctuation('''`'''  )  )
       self.assertTrue(_is_punctuation('''.'''  )  )

       self.assertFalse(_is_punctuation('''A'''  )  )
       self.assertFalse(_is_punctuation(''' '''  )  )







      @slow
      def       __lowerCAmelCase					(      self      :						List[str]  )    ->List[str]:




       """simple docstring"""


       a		  =      self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

       a		  =      tokenizer.encode('''sequence builders''' ,					add_special_tokens=__UpperCAmelCase  )
       a		  =      tokenizer.encode('''multi-sequence build''' ,					add_special_tokens=__UpperCAmelCase  )

       a		  =      tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase  )
       a		  =      tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase ,					__UpperCAmelCase  )

       assert encoded_sentence == text + [102]
       assert encoded_pair == text + [102] + text_a + [102]


