





import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import MaskaFormerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import MaskaFormerForUniversalSegmentation, MaskaFormerModel

     if is_vision_available():
          from transformers import MaskaFormerImageProcessor

if is_vision_available():
     from PIL import Image







class 				__lowerCAmelCase       :







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__=2		,      lowerCAmelCase__=True		,      lowerCAmelCase__=False		,      lowerCAmelCase__=1_0		,      lowerCAmelCase__=3		,      lowerCAmelCase__=3_2 * 8		,      lowerCAmelCase__=3_2 * 8		,      lowerCAmelCase__=4		,      lowerCAmelCase__=6_4		,      )   ->						List[str]:



        '''simple docstring'''




        a__	:					int										=parent
        a__	:					List[Any]										=batch_size
        a__	:					int										=is_training
        a__	:					Optional[Any]										=use_auxiliary_loss
        a__	:					int										=num_queries
        a__	:					Any										=num_channels
        a__	:					Any										=min_size
        a__	:					Optional[Any]										=max_size
        a__	:					List[str]										=num_labels
        a__	:					Dict										=hidden_dim
        a__	:					Dict										=hidden_dim







      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        a__	:					List[Any]										=floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]							).to(
            lowerCAmelCase__							)

        a__	:					Tuple										=torch.ones([self.batch_size, self.min_size, self.max_size]		,      device=lowerCAmelCase__							)

        a__	:					Dict										=(
            torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]		,      device=lowerCAmelCase__							) > 0.5
        ).float()
        a__	:					Dict										=(torch.rand((self.batch_size, self.num_labels)		,      device=lowerCAmelCase__							) > 0.5).long()

        a__	:					str										=self.get_config()
        return config, pixel_values, pixel_mask, mask_labels, class_labels







      def    _lowercase  (    self							)   ->						Dict:



        '''simple docstring'''




        a__	:					Optional[Any]										=MaskaFormerConfig(
            hidden_size=self.hidden_dim		,      )
        a__	:					Optional[Any]										=self.num_queries
        a__	:					int										=self.num_labels

        a__	:					Optional[Any]										=[1, 1, 1, 1]
        a__	:					Dict										=self.num_channels

        a__	:					List[str]										=6_4
        a__	:					Tuple										=1_2_8
        a__	:					List[str]										=self.hidden_dim
        a__	:					str										=self.hidden_dim
        a__	:					int										=self.hidden_dim
        return config







      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        a__					,							a__					,							a__					,							a__					,							a__	:					int										=self.prepare_config_and_inputs()
        a__	:					Any										={"pixel_values": pixel_values, "pixel_mask": pixel_mask}
        return config, inputs_dict







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						str:



        '''simple docstring'''




        a__	:					Optional[Any]										=output.encoder_hidden_states
        a__	:					str										=output.pixel_decoder_hidden_states
        a__	:					Dict										=output.transformer_decoder_hidden_states

        self.parent.assertTrue(len(lowerCAmelCase__							)		,      len(config.backbone_config.depths							)							)
        self.parent.assertTrue(len(lowerCAmelCase__							)		,      len(config.backbone_config.depths							)							)
        self.parent.assertTrue(len(lowerCAmelCase__							)		,      config.decoder_layers							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__=False							)   ->						List[Any]:



        '''simple docstring'''




        with torch.no_grad():
          a__	:					str										=MaskaFormerModel(config=lowerCAmelCase__							)
          model.to(lowerCAmelCase__							)
          model.eval()

          a__	:					Union[str, Any]										=model(pixel_values=lowerCAmelCase__		,      pixel_mask=lowerCAmelCase__							)
          a__	:					Optional[int]										=model(lowerCAmelCase__		,      output_hidden_states=lowerCAmelCase__							)

        self.parent.assertEqual(
            output.transformer_decoder_last_hidden_state.shape		,      (self.batch_size, self.num_queries, self.hidden_dim)		,      )
        # let's ensure the other two hidden state exists
        self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None							)
        self.parent.assertTrue(output.encoder_last_hidden_state is not None							)

        if output_hidden_states:
          self.check_output_hidden_state(lowerCAmelCase__		,      lowerCAmelCase__							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Dict:



        '''simple docstring'''




        a__	:					Any										=MaskaFormerForUniversalSegmentation(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()

        def comm_check_on_output(lowerCAmelCase__							):
          # let's still check that all the required stuff is there
          self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None							)
          self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None							)
          self.parent.assertTrue(result.encoder_last_hidden_state is not None							)
          # okay, now we need to check the logits shape
          # due to the encoder compression, masks have a //4 spatial size
          self.parent.assertEqual(
              result.masks_queries_logits.shape		,      (self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)		,      )
          # + 1 for null class
          self.parent.assertEqual(
              result.class_queries_logits.shape		,      (self.batch_size, self.num_queries, self.num_labels + 1)							)

        with torch.no_grad():
          a__	:					Optional[Any]										=model(pixel_values=lowerCAmelCase__		,      pixel_mask=lowerCAmelCase__							)
          a__	:					Any										=model(lowerCAmelCase__							)

          comm_check_on_output(lowerCAmelCase__							)

          a__	:					Optional[int]										=model(
              pixel_values=lowerCAmelCase__		,      pixel_mask=lowerCAmelCase__		,      mask_labels=lowerCAmelCase__		,      class_labels=lowerCAmelCase__							)

        comm_check_on_output(lowerCAmelCase__							)

        self.parent.assertTrue(result.loss is not None							)
        self.parent.assertEqual(result.loss.shape		,      torch.Size([1]							)							)







@require_torch
class 				__lowerCAmelCase       (    UpperCamelCase__						,				UpperCamelCase__						,				unittest.TestCase):
      _lowercase				:					Optional[int]						=			(MaskaFormerModel, MaskaFormerForUniversalSegmentation) if is_torch_available() else ()
      _lowercase				:					Union[str, Any]						=			{"""feature-extraction""": MaskaFormerModel} if is_torch_available() else {}

      _lowercase				:					str						=			False
      _lowercase				:					List[Any]						=			False
      _lowercase				:					List[str]						=			False
      _lowercase				:					int						=			False







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					Optional[Any]										=MaskaFormerModelTester(self							)
        a__	:					Optional[Any]										=ConfigTester(self		,      config_class=lowerCAmelCase__		,      has_text_modality=lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        a__					,							a__	:					Union[str, Any]										=self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.create_and_check_maskaformer_model(lowerCAmelCase__		,      **lowerCAmelCase__		,      output_hidden_states=lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        a__	:					List[str]										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_maskaformer_instance_segmentation_head_model(*lowerCAmelCase__							)







      @unittest.skip(reason="Mask2Former does not use inputs_embeds"							)
      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        pass







      @unittest.skip(reason="Mask2Former does not have a get_input_embeddings method"							)
      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        pass







      @unittest.skip(reason="Mask2Former is not a generative model"							)
      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        pass







      @unittest.skip(reason="Mask2Former does not use token embeddings"							)
      def    _lowercase  (    self							)   ->						List[Any]:



        '''simple docstring'''




        pass







      @require_torch_multi_gpu
      @unittest.skip(
          reason="Mask2Former has some layers using `add_module` which doesn't work well with `nn.DataParallel`"							)
      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        pass







      @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."							)
      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        pass







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__					,							a__	:					int										=self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__	:					Dict										=model_class(lowerCAmelCase__							)
          a__	:					List[str]										=inspect.signature(model.forward							)
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          a__	:					Dict										=[*signature.parameters.keys()]

          a__	:					List[str]										=["pixel_values"]
          self.assertListEqual(arg_names[:1]		,      lowerCAmelCase__							)







      @slow
      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        for model_name in ["facebook/mask2former-swin-small-coco-instance"]:
          a__	:					Optional[int]										=MaskaFormerModel.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					List[str]										=(self.model_tester.min_size,) * 2
        a__	:					Any										={
            "pixel_values": torch.randn((2, 3, *size)		,      device=lowerCAmelCase__							),
            "mask_labels": torch.randn((2, 1_0, *size)		,      device=lowerCAmelCase__							),
            "class_labels": torch.zeros(2		,      1_0		,      device=lowerCAmelCase__							).long(),
        }
        a__	:					Any										=self.model_tester.get_config()

        a__	:					str										=MaskaFormerForUniversalSegmentation(lowerCAmelCase__							).to(lowerCAmelCase__							)
        a__	:					Any										=model(**lowerCAmelCase__							)
        self.assertTrue(outputs.loss is not None							)







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__					,							a__	:					Tuple										=self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.create_and_check_maskaformer_model(lowerCAmelCase__		,      **lowerCAmelCase__		,      output_hidden_states=lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__					,							a__	:					Union[str, Any]										=self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          a__	:					List[Any]										=model_class(lowerCAmelCase__							).to(lowerCAmelCase__							)
          a__	:					Optional[int]										=model(**lowerCAmelCase__		,      output_attentions=lowerCAmelCase__							)
          self.assertTrue(outputs.attentions is not None							)







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        if not self.model_tester.is_training:
          return

        a__	:					Optional[Any]										=self.all_model_classes[1]
        a__					,							a__					,							a__					,							a__					,							a__	:					Optional[int]										=self.model_tester.prepare_config_and_inputs()

        a__	:					str										=model_class(lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.train()

        a__	:					List[str]										=model(lowerCAmelCase__		,      mask_labels=lowerCAmelCase__		,      class_labels=lowerCAmelCase__							).loss
        loss.backward()







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					Dict										=self.all_model_classes[1]
        a__					,							a__					,							a__					,							a__					,							a__	:					Dict										=self.model_tester.prepare_config_and_inputs()
        a__	:					Tuple										=True
        a__	:					int										=True

        a__	:					List[Any]										=model_class(lowerCAmelCase__							).to(lowerCAmelCase__							)
        model.train()

        a__	:					List[str]										=model(lowerCAmelCase__		,      mask_labels=lowerCAmelCase__		,      class_labels=lowerCAmelCase__							)

        a__	:					Optional[int]										=outputs.encoder_hidden_states[0]
        encoder_hidden_states.retain_grad()

        a__	:					Optional[Any]										=outputs.pixel_decoder_hidden_states[0]
        pixel_decoder_hidden_states.retain_grad()

        a__	:					Tuple										=outputs.transformer_decoder_hidden_states[0]
        transformer_decoder_hidden_states.retain_grad()

        a__	:					Optional[Any]										=outputs.attentions[0]
        attentions.retain_grad()

        outputs.loss.backward(retain_graph=lowerCAmelCase__							)

        self.assertIsNotNone(encoder_hidden_states.grad							)
        self.assertIsNotNone(pixel_decoder_hidden_states.grad							)
        self.assertIsNotNone(transformer_decoder_hidden_states.grad							)
        self.assertIsNotNone(attentions.grad							)
UpperCAmelCase	:	int		     =       1E-4
def      _A       (		):





  """simple docstring"""



  a__	:					int										=Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"						)
  return image







@require_vision
@slow
class 				__lowerCAmelCase       (    unittest.TestCase):







      @cached_property
      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        return "facebook/mask2former-swin-small-coco-instance"







      @cached_property
      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        return MaskaFormerImageProcessor.from_pretrained(self.model_checkpoints							) if is_vision_available() else None







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					Optional[int]										=MaskaFormerModel.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase__							)
        a__	:					Tuple										=self.default_image_processor
        a__	:					Tuple										=prepare_img()
        a__	:					int										=image_processor(lowerCAmelCase__		,      return_tensors="pt"							).to(lowerCAmelCase__							)
        a__	:					List[Any]										=inputs["pixel_values"].shape
        # check size is divisible by 32
        self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0							)
        # check size
        self.assertEqual(lowerCAmelCase__		,      (1, 3, 3_8_4, 3_8_4)							)

        with torch.no_grad():
          a__	:					Tuple										=model(**lowerCAmelCase__							)

        a__	:					Union[str, Any]										=torch.tensor(
            [[-0.27_90, -1.07_17, -1.16_68], [-0.51_28, -0.31_28, -0.49_87], [-0.58_32, 0.19_71, -0.01_97]]							).to(lowerCAmelCase__							)
        self.assertTrue(
            torch.allclose(
                outputs.encoder_last_hidden_state[0, 0, :3, :3]		,      lowerCAmelCase__		,      atol=lowerCAmelCase__							)							)

        a__	:					List[Any]										=torch.tensor(
            [[0.89_73, 1.18_47, 1.17_76], [1.19_34, 1.50_40, 1.51_28], [1.11_53, 1.44_86, 1.49_51]]							).to(lowerCAmelCase__							)
        self.assertTrue(
            torch.allclose(
                outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]		,      lowerCAmelCase__		,      atol=lowerCAmelCase__							)							)

        a__	:					str										=torch.tensor(
            [[2.11_52, 1.70_00, -0.86_03], [1.58_08, 1.80_04, -0.93_53], [1.60_43, 1.74_95, -0.59_99]]							).to(lowerCAmelCase__							)
        self.assertTrue(
            torch.allclose(
                outputs.transformer_decoder_last_hidden_state[0, :3, :3]		,      lowerCAmelCase__		,      atol=lowerCAmelCase__							)							)







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					Optional[int]										=MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase__							).eval()
        a__	:					List[str]										=self.default_image_processor
        a__	:					Optional[int]										=prepare_img()
        a__	:					Optional[int]										=image_processor(lowerCAmelCase__		,      return_tensors="pt"							).to(lowerCAmelCase__							)
        a__	:					Union[str, Any]										=inputs["pixel_values"].shape
        # check size is divisible by 32
        self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0							)
        # check size
        self.assertEqual(lowerCAmelCase__		,      (1, 3, 3_8_4, 3_8_4)							)

        with torch.no_grad():
          a__	:					Tuple										=model(**lowerCAmelCase__							)
        # masks_queries_logits
        a__	:					Tuple										=outputs.masks_queries_logits
        self.assertEqual(
            masks_queries_logits.shape		,      (1, model.config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)							)
        a__	:					int										=[
            [-8.78_39, -9.00_56, -8.81_21],
            [-7.41_04, -7.03_13, -6.54_01],
            [-6.61_05, -6.34_27, -6.46_75],
        ]
        a__	:					List[str]										=torch.tensor(lowerCAmelCase__							).to(lowerCAmelCase__							)
        self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]		,      lowerCAmelCase__		,      atol=lowerCAmelCase__							)							)
        # class_queries_logits
        a__	:					str										=outputs.class_queries_logits
        self.assertEqual(class_queries_logits.shape		,      (1, model.config.num_queries, model.config.num_labels + 1)							)
        a__	:					Any										=torch.tensor(
            [
                [1.83_24, -8.08_35, -4.19_22],
                [0.84_50, -9.00_50, -3.60_53],
                [0.30_45, -7.72_93, -3.02_75],
            ]							).to(lowerCAmelCase__							)
        self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]		,      lowerCAmelCase__		,      atol=lowerCAmelCase__							)							)







      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=MaskaFormerForUniversalSegmentation.from_pretrained(self.model_checkpoints							).to(lowerCAmelCase__							).eval()
        a__	:					Union[str, Any]										=self.default_image_processor

        a__	:					Dict										=image_processor(
            [np.zeros((3, 8_0_0, 1_3_3_3)							), np.zeros((3, 8_0_0, 1_3_3_3)							)]		,      segmentation_maps=[np.zeros((3_8_4, 3_8_4)							).astype(np.floataa							), np.zeros((3_8_4, 3_8_4)							).astype(np.floataa							)]		,      return_tensors="pt"		,      )

        a__	:					int										=inputs["pixel_values"].to(lowerCAmelCase__							)
        a__	:					str										=[el.to(lowerCAmelCase__							) for el in inputs["mask_labels"]]
        a__	:					Any										=[el.to(lowerCAmelCase__							) for el in inputs["class_labels"]]

        with torch.no_grad():
          a__	:					Tuple										=model(**lowerCAmelCase__							)

        self.assertTrue(outputs.loss is not None							)












"""simple docstring"""


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
     if is_torch_available():
          import torch
     if is_tf_available():
          import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


A_						=				logging.get_logger(__name__)

A_						=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A_						=				{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

A_						=				{
    '''Salesforce/codegen-350M-mono''': 20_48,
}






class 					lowercase(     __a ):







 '''simple docstring'''





 lowercase__ 					=					VOCAB_FILES_NAMES
 lowercase__ 					=					PRETRAINED_VOCAB_FILES_MAP
 lowercase__ 					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__ 					=					["input_ids", "attention_mask"]
 lowercase__ 					=					CodeGenTokenizer


 def __init__(				self:		Union[str, Any],      a_:		List[Any]=None,      a_:		str=None,      a_:		str=None,      a_:		Dict="<|endoftext|>",      a_:		Tuple="<|endoftext|>",      a_:		str="<|endoftext|>",      a_:		List[Any]=False,      **a_:		List[str],      ):




   '''simple docstring'''



   super().__init__(
       a_,      a_,      tokenizer_file=a_,      unk_token=a_,      bos_token=a_,      eos_token=a_,      add_prefix_space=a_,      **a_,      )

   if kwargs.pop("""add_bos_token""",      a_		):
     _snake_case							:		str								=  kwargs.pop("""name_or_path""",      """"""		)
     raise ValueError(
         """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
         """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
         f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
         f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
         """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
         """ so that the fast tokenizer works correctly."""		)

   _snake_case							:		Tuple								=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
   if pre_tok_state.get("""add_prefix_space""",      a_		) != add_prefix_space:
     _snake_case							:		Dict								=  getattr(a_,      pre_tok_state.pop("""type"""		)		)
     _snake_case							:		Dict								=  add_prefix_space
     _snake_case							:		str								=  pre_tok_class(**a_		)

   _snake_case							:		List[Any]								=  add_prefix_space


 def  UpperCamelCase_   (				self:		Any,      *a_:		Any,      **a_:		int		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  kwargs.get("""is_split_into_words""",      a_		)
   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		Optional[Any],      *a_:		Any,      **a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		Dict								=  kwargs.get("""is_split_into_words""",      a_		)

   assert self.add_prefix_space or not is_split_into_words, (
       f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*a_,      **a_		)


 def  UpperCamelCase_   (				self:		Optional[int],      a_:		str,      a_:		Optional[str] = None		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  self._tokenizer.model.save(a_,      name=a_		)
   return tuple(a_		)


 def  UpperCamelCase_   (				self:		str,      a_:		Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],      a_:		bool = False,      a_:		bool = None,      a_:		Optional[List[str]] = None,      **a_:		List[str],      ):




   '''simple docstring'''



   _snake_case							:		Any								=  super().decode(
       token_ids=a_,      skip_special_tokens=a_,      clean_up_tokenization_spaces=a_,      **a_,      )

   if truncate_before_pattern is not None and len(a_		) > 0:
     _snake_case							:		List[str]								=  self.truncate(a_,      a_		)

   return decoded_text


 def  UpperCamelCase_   (				self:		Dict,      a_:		Tuple,      a_:		Optional[Any]		):




   '''simple docstring'''



   def find_re(a_:		Dict,      a_:		str,      a_:		Union[str, Any]		):
     _snake_case							:		Any								=  pattern.search(a_,      a_		)
     return m.start() if m else -1

   _snake_case							:		Tuple								=  [re.compile(a_,      re.MULTILINE		) for pattern in truncate_before_pattern]

   _snake_case							:		List[Any]								=  list(re.finditer("""^print""",      a_,      re.MULTILINE		)		)

   if len(a_		) > 1:
     _snake_case							:		int								=  completion[: prints[1].start()]

   _snake_case							:		List[str]								=  list(re.finditer("""^def""",      a_,      re.MULTILINE		)		)

   if len(a_		) > 1:
     _snake_case							:		List[Any]								=  completion[: defs[1].start()]

   _snake_case							:		int								=  0

   _snake_case							:		List[Any]								=  [
       pos for pos in [find_re(a_,      a_,      a_		) for terminal in terminals] if pos != -1
   ]

   if len(a_		) > 0:
     return completion[: min(a_		)]
   else:
     return completion


