




"""simple docstring"""


import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
     from transformers.models.esm.modeling_esm import (
         ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
         EsmEmbeddings,
         create_position_ids_from_input_ids,
     )






class 					lowercase:







 '''simple docstring'''





 def __init__(				self:		List[Any],      a_:		Optional[Any],      a_:		List[Any]=13,      a_:		List[Any]=7,      a_:		Tuple=False,      a_:		str=True,      a_:		str=False,      a_:		List[str]=True,      a_:		Dict=33,      a_:		Any=32,      a_:		Tuple=5,      a_:		List[Any]=4,      a_:		Any=37,      a_:		str="gelu",      a_:		Tuple=0.1,      a_:		Union[str, Any]=0.1,      a_:		Dict=512,      a_:		str=16,      a_:		str=2,      a_:		Tuple=0.02,      a_:		Optional[int]=3,      a_:		str=4,      a_:		Any=None,      ):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  parent
   _snake_case							:		Optional[Any]								=  batch_size
   _snake_case							:		int								=  seq_length
   _snake_case							:		Optional[int]								=  is_training
   _snake_case							:		List[str]								=  use_input_mask
   _snake_case							:		List[Any]								=  use_token_type_ids
   _snake_case							:		Any								=  use_labels
   _snake_case							:		List[Any]								=  vocab_size
   _snake_case							:		Any								=  hidden_size
   _snake_case							:		int								=  num_hidden_layers
   _snake_case							:		str								=  num_attention_heads
   _snake_case							:		Optional[int]								=  intermediate_size
   _snake_case							:		Union[str, Any]								=  hidden_act
   _snake_case							:		Tuple								=  hidden_dropout_prob
   _snake_case							:		Union[str, Any]								=  attention_probs_dropout_prob
   _snake_case							:		Optional[Any]								=  max_position_embeddings
   _snake_case							:		Union[str, Any]								=  type_vocab_size
   _snake_case							:		Any								=  type_sequence_label_size
   _snake_case							:		str								=  initializer_range
   _snake_case							:		Tuple								=  num_labels
   _snake_case							:		Any								=  num_choices
   _snake_case							:		Optional[int]								=  scope


 def  UpperCamelCase_   (				self:		Any		):




   '''simple docstring'''



   _snake_case							:		Dict								=  ids_tensor([self.batch_size, self.seq_length],      self.vocab_size		)

   _snake_case							:		Any								=  None
   if self.use_input_mask:
     _snake_case							:		Tuple								=  random_attention_mask([self.batch_size, self.seq_length]		)

   _snake_case							:		List[Any]								=  None
   _snake_case							:		Any								=  None
   _snake_case							:		Optional[int]								=  None
   if self.use_labels:
     _snake_case							:		Optional[Any]								=  ids_tensor([self.batch_size],      self.type_sequence_label_size		)
     _snake_case							:		List[Any]								=  ids_tensor([self.batch_size, self.seq_length],      self.num_labels		)
     _snake_case							:		Tuple								=  ids_tensor([self.batch_size],      self.num_choices		)

   _snake_case							:		Any								=  self.get_config()

   return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   return EsmConfig(
       vocab_size=self.vocab_size,      hidden_size=self.hidden_size,      pad_token_id=1,      num_hidden_layers=self.num_hidden_layers,      num_attention_heads=self.num_attention_heads,      intermediate_size=self.intermediate_size,      hidden_act=self.hidden_act,      hidden_dropout_prob=self.hidden_dropout_prob,      attention_probs_dropout_prob=self.attention_probs_dropout_prob,      max_position_embeddings=self.max_position_embeddings,      type_vocab_size=self.type_vocab_size,      initializer_range=self.initializer_range,      )


 def  UpperCamelCase_   (				self:		int,      a_:		Union[str, Any],      a_:		Dict,      a_:		List[str],      a_:		Any,      a_:		Any,      a_:		Any		):




   '''simple docstring'''



   _snake_case							:		Dict								=  EsmModel(config=a_		)
   model.to(a_		)
   model.eval()
   _snake_case							:		List[str]								=  model(a_,      attention_mask=a_		)
   _snake_case							:		Any								=  model(a_		)
   _snake_case							:		Optional[Any]								=  model(a_		)

   self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.hidden_size)		)
   self.parent.assertEqual(result.pooler_output.shape,      (self.batch_size, self.hidden_size)		)


 def  UpperCamelCase_   (				self:		Tuple,      a_:		int,      a_:		Dict,      a_:		Optional[Any],      a_:		str,      a_:		Union[str, Any],      a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  EsmForMaskedLM(config=a_		)
   model.to(a_		)
   model.eval()
   _snake_case							:		Optional[Any]								=  model(a_,      attention_mask=a_,      labels=a_		)
   self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.vocab_size)		)


 def  UpperCamelCase_   (				self:		List[Any],      a_:		Optional[Any],      a_:		Any,      a_:		Any,      a_:		str,      a_:		Union[str, Any],      a_:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.num_labels
   _snake_case							:		Tuple								=  EsmForTokenClassification(config=a_		)
   model.to(a_		)
   model.eval()
   _snake_case							:		List[Any]								=  model(a_,      attention_mask=a_,      labels=a_		)
   self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.num_labels)		)


 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.prepare_config_and_inputs()
   (
       (
       _snake_case
   )		,					(
       _snake_case
   )		,					(
       _snake_case
   )		,					(
       _snake_case
   )		,					(
       _snake_case
   )		,					(
       _snake_case
   )		,					
   )							:		str								=  config_and_inputs
   _snake_case							:		Optional[Any]								=  {"""input_ids""": input_ids, """attention_mask""": input_mask}
   return config, inputs_dict







@require_torch
class 					lowercase(     __a       ,			__a       ,			unittest.TestCase ):







 '''simple docstring'''





 lowercase__ 					=					False

 lowercase__ 					=					(
     (
         EsmForMaskedLM,
         EsmModel,
         EsmForSequenceClassification,
         EsmForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 lowercase__ 					=					()
 lowercase__ 					=					(
     {
         "feature-extraction": EsmModel,
         "fill-mask": EsmForMaskedLM,
         "text-classification": EsmForSequenceClassification,
         "token-classification": EsmForTokenClassification,
         "zero-shot": EsmForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 lowercase__ 					=					True


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		str								=  EsmModelTester(self		)
   _snake_case							:		Union[str, Any]								=  ConfigTester(self,      config_class=a_,      hidden_size=37		)


 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   self.config_tester.run_common_tests()


 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_model(*a_		)


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  self.model_tester.prepare_config_and_inputs()
   for type in ["absolute", "relative_key", "relative_key_query"]:
     _snake_case							:		str								=  type
     self.model_tester.create_and_check_model(*a_		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_masked_lm(*a_		)


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_token_classification(*a_		)


 @slow
 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     _snake_case							:		Tuple								=  EsmModel.from_pretrained(a_		)
     self.assertIsNotNone(a_		)


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		Any								=  self.model_tester.prepare_config_and_inputs()[0]
   _snake_case							:		Union[str, Any]								=  EsmEmbeddings(config=a_		)

   _snake_case							:		Union[str, Any]								=  torch.as_tensor([[12, 31, 13, model.padding_idx]]		)
   _snake_case							:		int								=  torch.as_tensor(
       [
           [
               0 + model.padding_idx + 1,
               1 + model.padding_idx + 1,
               2 + model.padding_idx + 1,
               model.padding_idx,
           ]
       ]		)
   _snake_case							:		List[Any]								=  create_position_ids_from_input_ids(a_,      model.padding_idx		)
   self.assertEqual(position_ids.shape,      expected_positions.shape		)
   self.assertTrue(torch.all(torch.eq(a_,      a_		)		)		)


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		Any								=  self.model_tester.prepare_config_and_inputs()[0]
   _snake_case							:		Optional[int]								=  EsmEmbeddings(config=a_		)

   _snake_case							:		int								=  torch.empty(2,      4,      30		)
   _snake_case							:		Any								=  [
       0 + embeddings.padding_idx + 1,
       1 + embeddings.padding_idx + 1,
       2 + embeddings.padding_idx + 1,
       3 + embeddings.padding_idx + 1,
   ]
   _snake_case							:		int								=  torch.as_tensor([expected_single_positions, expected_single_positions]		)
   _snake_case							:		List[Any]								=  embeddings.create_position_ids_from_inputs_embeds(a_		)
   self.assertEqual(position_ids.shape,      expected_positions.shape		)
   self.assertTrue(torch.all(torch.eq(a_,      a_		)		)		)


 @unittest.skip("""Esm does not support embedding resizing"""		)
 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   pass


 @unittest.skip("""Esm does not support embedding resizing"""		)
 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   pass


 @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""		)
 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   pass







@require_torch
class 					lowercase(     __a ):







 '''simple docstring'''





 @slow
 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   with torch.no_grad():
     _snake_case							:		Optional[Any]								=  EsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""		)
     model.eval()
     _snake_case							:		Union[str, Any]								=  torch.tensor([[0, 1, 2, 3, 4, 5]]		)
     _snake_case							:		Optional[int]								=  model(a_		)[0]

     _snake_case							:		Tuple								=  33

     _snake_case							:		str								=  torch.Size((1, 6, vocab_size)		)
     self.assertEqual(output.shape,      a_		)

     _snake_case							:		str								=  torch.tensor(
         [[[8.9_215, -10.5_898, -6.4_671], [-6.3_967, -13.9_114, -1.1_212], [-7.7_812, -13.9_516, -3.7_406]]]		)
     self.assertTrue(torch.allclose(output[:, :3, :3],      a_,      atol=1E-4		)		)


 @slow
 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   with torch.no_grad():
     _snake_case							:		Tuple								=  EsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""		)
     model.eval()

     _snake_case							:		Any								=  torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]		)
     _snake_case							:		Optional[int]								=  model(a_		)[0]
     # compare the actual values for a slice.
     _snake_case							:		Union[str, Any]								=  torch.tensor(
         [[[0.1_444, 0.5_413, 0.3_248], [0.3_034, 0.0_053, 0.3_108], [0.3_228, -0.2_499, 0.3_415]]]		)
     self.assertTrue(torch.allclose(output[:, :3, :3],      a_,      atol=1E-4		)		)








"""simple docstring"""


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
     if is_torch_available():
          import torch
     if is_tf_available():
          import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


A_						=				logging.get_logger(__name__)

A_						=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A_						=				{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

A_						=				{
    '''Salesforce/codegen-350M-mono''': 20_48,
}






class 					lowercase(     __a ):







 '''simple docstring'''





 lowercase__ 					=					VOCAB_FILES_NAMES
 lowercase__ 					=					PRETRAINED_VOCAB_FILES_MAP
 lowercase__ 					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__ 					=					["input_ids", "attention_mask"]
 lowercase__ 					=					CodeGenTokenizer


 def __init__(				self:		Union[str, Any],      a_:		List[Any]=None,      a_:		str=None,      a_:		str=None,      a_:		Dict="<|endoftext|>",      a_:		Tuple="<|endoftext|>",      a_:		str="<|endoftext|>",      a_:		List[Any]=False,      **a_:		List[str],      ):




   '''simple docstring'''



   super().__init__(
       a_,      a_,      tokenizer_file=a_,      unk_token=a_,      bos_token=a_,      eos_token=a_,      add_prefix_space=a_,      **a_,      )

   if kwargs.pop("""add_bos_token""",      a_		):
     _snake_case							:		str								=  kwargs.pop("""name_or_path""",      """"""		)
     raise ValueError(
         """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
         """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
         f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
         f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
         """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
         """ so that the fast tokenizer works correctly."""		)

   _snake_case							:		Tuple								=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
   if pre_tok_state.get("""add_prefix_space""",      a_		) != add_prefix_space:
     _snake_case							:		Dict								=  getattr(a_,      pre_tok_state.pop("""type"""		)		)
     _snake_case							:		Dict								=  add_prefix_space
     _snake_case							:		str								=  pre_tok_class(**a_		)

   _snake_case							:		List[Any]								=  add_prefix_space


 def  UpperCamelCase_   (				self:		Any,      *a_:		Any,      **a_:		int		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  kwargs.get("""is_split_into_words""",      a_		)
   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		Optional[Any],      *a_:		Any,      **a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		Dict								=  kwargs.get("""is_split_into_words""",      a_		)

   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		Optional[int],      a_:		str,      a_:		Optional[str] = None		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  self._tokenizer.model.save(a_,      name=a_		)
   return tuple(a_		)


 def  UpperCamelCase_   (				self:		str,      a_:		Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],      a_:		bool = False,      a_:		bool = None,      a_:		Optional[List[str]] = None,      **a_:		List[str],      ):




   '''simple docstring'''



   _snake_case							:		Any								=  super().decode(
       token_ids=a_,      skip_special_tokens=a_,      clean_up_tokenization_spaces=a_,      **a_,      )

   if truncate_before_pattern is not None and len(a_		) > 0:
     _snake_case							:		List[str]								=  self.truncate(a_,      a_		)

   return decoded_text


 def  UpperCamelCase_   (				self:		Dict,      a_:		Tuple,      a_:		Optional[Any]		):




   '''simple docstring'''



   def find_re(a_:		Dict,      a_:		str,      a_:		Union[str, Any]		):
     _snake_case							:		Any								=  pattern.search(a_,      a_		)
     return m.start() if m else -1

   _snake_case							:		Tuple								=  [re.compile(a_,      re.MULTILINE		) for pattern in truncate_before_pattern]

   _snake_case							:		List[Any]								=  list(re.finditer("""^print""",      a_,      re.MULTILINE		)		)

   if len(a_		) > 1:
     _snake_case							:		int								=  completion[: prints[1].start()]

   _snake_case							:		List[str]								=  list(re.finditer("""^def""",      a_,      re.MULTILINE		)		)

   if len(a_		) > 1:
     _snake_case							:		List[Any]								=  completion[: defs[1].start()]

   _snake_case							:		int								=  0

   _snake_case							:		List[Any]								=  [
       pos for pos in [find_re(a_,      a_,      a_		) for terminal in terminals] if pos != -1
   ]

   if len(a_		) > 0:
     return completion[: min(a_		)]
   else:
     return completion


