





import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A__:  Dict        =     get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
   from transformers.models.plbart.modeling_plbart import shift_tokens_right

A__:  Any        =     5_0003
A__:  Tuple        =     5_0002




@require_sentencepiece
@require_tokenizers
class     _a  (  a__		,       unittest.TestCase):

      """simple docstring"""


      UpperCamelCase__   						=    PLBartTokenizer
      UpperCamelCase__   						=    None
      UpperCamelCase__   						=    False






      def        UpperCAmelCase_    (						self:       str ):
          '''simple docstring'''


          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCamelCase__:     List[Any]     							=			PLBartTokenizer(SCREAMING_SNAKE_CASE__     ,  language_codes="base"     ,  keep_accents=SCREAMING_SNAKE_CASE__ )
          tokenizer.save_pretrained(self.tmpdirname )






      def        UpperCAmelCase_    (						self:       int ):
          '''simple docstring'''


          UpperCamelCase__:     Tuple     							=			PLBartTokenizer(SCREAMING_SNAKE_CASE__     ,  language_codes="base"     ,  keep_accents=SCREAMING_SNAKE_CASE__ )

          UpperCamelCase__:     Tuple     							=			tokenizer.tokenize("This is a test" )
          self.assertListEqual(SCREAMING_SNAKE_CASE__     ,  ["▁This", "▁is", "▁a", "▁t", "est"] )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__ )     ,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,  )

          UpperCamelCase__:     int     							=			tokenizer.tokenize("I was born in 92000, and this is falsé." )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "é",
                  ".",
              ]     ,  )
          UpperCamelCase__:     int     							=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__ )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]     ,  )

          UpperCamelCase__:     Any     							=			tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__ )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "<unk>",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "<unk>",
                  ".",
              ]     ,  )

          UpperCamelCase__:     Dict     							=			tokenizer.vocab_size
          UpperCamelCase__:     Union[str, Any]     							=			[tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__ ) for x in range(end - 4     ,  SCREAMING_SNAKE_CASE__ )]

          self.assertListEqual(SCREAMING_SNAKE_CASE__     ,  ["__java__", "__python__", "__en_XX__", "<mask>"] )

          UpperCamelCase__:     str     							=			"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
          UpperCamelCase__:     Tuple     							=			tokenizer(SCREAMING_SNAKE_CASE__ ).input_ids
          self.assertEqual(
              tokenizer.decode(SCREAMING_SNAKE_CASE__     ,  skip_special_tokens=SCREAMING_SNAKE_CASE__     ,  clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__ )     ,  SCREAMING_SNAKE_CASE__     ,  )






      def        UpperCAmelCase_    (						self:       Union[str, Any] ):
          '''simple docstring'''


          UpperCamelCase__:     Any     							=			PLBartTokenizer(SCREAMING_SNAKE_CASE__     ,  language_codes="multi"     ,  keep_accents=SCREAMING_SNAKE_CASE__ )

          UpperCamelCase__:     Tuple     							=			tokenizer.tokenize("This is a test" )
          self.assertListEqual(SCREAMING_SNAKE_CASE__     ,  ["▁This", "▁is", "▁a", "▁t", "est"] )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__ )     ,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,  )

          UpperCamelCase__:     Union[str, Any]     							=			tokenizer.tokenize("I was born in 92000, and this is falsé." )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "é",
                  ".",
              ]     ,  )
          UpperCamelCase__:     Tuple     							=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__ )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  value + tokenizer.fairseq_offset
                  for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
              ]     ,  )

          UpperCamelCase__:     Dict     							=			tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__ )
          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  [
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "<unk>",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "<unk>",
                  ".",
              ]     ,  )
          UpperCamelCase__:     List[Any]     							=			tokenizer.vocab_size
          UpperCamelCase__:     Dict     							=			[tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__ ) for x in range(end - 7     ,  SCREAMING_SNAKE_CASE__ )]

          self.assertListEqual(
              SCREAMING_SNAKE_CASE__     ,  ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"] )
          UpperCamelCase__:     str     							=			"""java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
          UpperCamelCase__:     str     							=			tokenizer(SCREAMING_SNAKE_CASE__ ).input_ids
          self.assertEqual(
              tokenizer.decode(SCREAMING_SNAKE_CASE__     ,  skip_special_tokens=SCREAMING_SNAKE_CASE__     ,  clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__ )     ,  SCREAMING_SNAKE_CASE__     ,  )




@require_torch
@require_sentencepiece
@require_tokenizers
class     _a  (  unittest.TestCase):

      """simple docstring"""


      UpperCamelCase__   						=    '''uclanlp/plbart-python-en_XX'''
      UpperCamelCase__   						=    [
          '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
          '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
      ]
      UpperCamelCase__   						=    [
          '''Returns the maximum value of a b c.''',
          '''Sums the values of a b c.''',
      ]
      UpperCamelCase__   						=    [
          134,
          5452,
          3_3460,
          3_3441,
          3_3463,
          3_3465,
          3_3463,
          3_3449,
          988,
          20,
          3_3456,
          19,
          3_3456,
          771,
          39,
          4258,
          889,
          3318,
          3_3441,
          3_3463,
          3_3465,
          3_3463,
          3_3449,
          2471,
          2,
          PYTHON_CODE,
      ]






      @classmethod
      def        UpperCAmelCase_    (						cls:       List[Any] ):
          '''simple docstring'''


          UpperCamelCase__:     PLBartTokenizer     							=			PLBartTokenizer.from_pretrained(
              cls.checkpoint_name     ,  language_codes="base"     ,  src_lang="python"     ,  tgt_lang="en_XX" )
          UpperCamelCase__:     Dict     							=			1
          return cls






      def        UpperCAmelCase_    (						self:       Optional[Any] ):
          '''simple docstring'''


          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]     ,  5_0001 )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]     ,  5_0002 )
          self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]     ,  5_0003 )






      def        UpperCAmelCase_    (						self:       Any ):
          '''simple docstring'''


          UpperCamelCase__:     Dict     							=			self.tokenizer.batch_encode_plus(self.src_text ).input_ids[0]
          self.assertListEqual(self.expected_src_tokens     ,  SCREAMING_SNAKE_CASE__ )






      def        UpperCAmelCase_    (						self:       Tuple ):
          '''simple docstring'''


          self.assertIn(SCREAMING_SNAKE_CASE__     ,  self.tokenizer.all_special_ids )
          UpperCamelCase__:     int     							=			[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
          UpperCamelCase__:     Tuple     							=			self.tokenizer.decode(SCREAMING_SNAKE_CASE__     ,  skip_special_tokens=SCREAMING_SNAKE_CASE__ )
          UpperCamelCase__:     int     							=			self.tokenizer.decode(generated_ids[1:]     ,  skip_special_tokens=SCREAMING_SNAKE_CASE__ )
          self.assertEqual(SCREAMING_SNAKE_CASE__     ,  SCREAMING_SNAKE_CASE__ )
          self.assertNotIn(self.tokenizer.eos_token     ,  SCREAMING_SNAKE_CASE__ )






      def        UpperCAmelCase_    (						self:       str ):
          '''simple docstring'''


          UpperCamelCase__:     Optional[Any]     							=			["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
          self.assertIsInstance(src_text[0]     ,  SCREAMING_SNAKE_CASE__ )
          UpperCamelCase__:     Tuple     							=			10
          UpperCamelCase__:     Dict     							=			self.tokenizer(SCREAMING_SNAKE_CASE__     ,  max_length=SCREAMING_SNAKE_CASE__     ,  truncation=SCREAMING_SNAKE_CASE__ ).input_ids[0]
          self.assertEqual(ids[-2]     ,  2 )
          self.assertEqual(ids[-1]     ,  SCREAMING_SNAKE_CASE__ )
          self.assertEqual(len(SCREAMING_SNAKE_CASE__ )     ,  SCREAMING_SNAKE_CASE__ )






      def        UpperCAmelCase_    (						self:       Optional[Any] ):
          '''simple docstring'''


          self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"] )     ,  [5_0004, 5_0001] )






      def        UpperCAmelCase_    (						self:       int ):
          '''simple docstring'''


          UpperCamelCase__:     Any     							=			tempfile.mkdtemp()
          UpperCamelCase__:     Any     							=			self.tokenizer.fairseq_tokens_to_ids
          self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__ )
          UpperCamelCase__:     List[Any]     							=			PLBartTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__ )
          self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,  SCREAMING_SNAKE_CASE__ )






      @require_torch
      def        UpperCAmelCase_    (						self:       str ):
          '''simple docstring'''


          UpperCamelCase__:     List[Any]     							=			self.tokenizer(self.src_text     ,  text_target=self.tgt_text     ,  padding=SCREAMING_SNAKE_CASE__     ,  return_tensors="pt" )
          UpperCamelCase__:     Union[str, Any]     							=			shift_tokens_right(batch["labels"]     ,  self.tokenizer.pad_token_id )

          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          self.assertEqual(batch.input_ids[1][-2:].tolist()     ,  [2, PYTHON_CODE] )
          self.assertEqual(batch.decoder_input_ids[1][0]     ,  SCREAMING_SNAKE_CASE__ )
          self.assertEqual(batch.decoder_input_ids[1][-1]     ,  2 )
          self.assertEqual(batch.labels[1][-2:].tolist()     ,  [2, EN_CODE] )






      @require_torch
      def        UpperCAmelCase_    (						self:       Optional[int] ):
          '''simple docstring'''


          UpperCamelCase__:     Optional[int]     							=			self.tokenizer(
              self.src_text     ,  text_target=self.tgt_text     ,  padding=SCREAMING_SNAKE_CASE__     ,  truncation=SCREAMING_SNAKE_CASE__     ,  max_length=len(self.expected_src_tokens )     ,  return_tensors="pt"     ,  )
          UpperCamelCase__:     Optional[Any]     							=			shift_tokens_right(batch["labels"]     ,  self.tokenizer.pad_token_id )

          self.assertIsInstance(SCREAMING_SNAKE_CASE__     ,  SCREAMING_SNAKE_CASE__ )

          self.assertEqual((2, 26)     ,  batch.input_ids.shape )
          self.assertEqual((2, 26)     ,  batch.attention_mask.shape )
          UpperCamelCase__:     Optional[Any]     							=			batch.input_ids.tolist()[0]
          self.assertListEqual(self.expected_src_tokens     ,  SCREAMING_SNAKE_CASE__ )
          self.assertEqual(2     ,  batch.decoder_input_ids[0, -1] )  # EOS
          # Test that special tokens are reset
          self.assertEqual(self.tokenizer.prefix_tokens     ,  [] )
          self.assertEqual(self.tokenizer.suffix_tokens     ,  [self.tokenizer.eos_token_id, PYTHON_CODE] )






      def        UpperCAmelCase_    (						self:       Union[str, Any] ):
          '''simple docstring'''


          UpperCamelCase__:     Dict     							=			self.tokenizer(self.src_text     ,  padding=SCREAMING_SNAKE_CASE__     ,  truncation=SCREAMING_SNAKE_CASE__     ,  max_length=3     ,  return_tensors="pt" )
          UpperCamelCase__:     int     							=			self.tokenizer(
              text_target=self.tgt_text     ,  padding=SCREAMING_SNAKE_CASE__     ,  truncation=SCREAMING_SNAKE_CASE__     ,  max_length=10     ,  return_tensors="pt" )
          UpperCamelCase__:     Optional[int]     							=			targets["""input_ids"""]
          UpperCamelCase__:     Dict     							=			shift_tokens_right(SCREAMING_SNAKE_CASE__     ,  self.tokenizer.pad_token_id )

          self.assertEqual(batch.input_ids.shape[1]     ,  3 )
          self.assertEqual(batch.decoder_input_ids.shape[1]     ,  10 )






      @require_torch
      def        UpperCAmelCase_    (						self:       Dict ):
          '''simple docstring'''


          UpperCamelCase__:     Union[str, Any]     							=			self.tokenizer._build_translation_inputs(
              "A test"     ,  return_tensors="pt"     ,  src_lang="en_XX"     ,  tgt_lang="java" )

          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE__ )     ,  {
                  # A, test, EOS, en_XX
                  "input_ids": [[150, 242, 2, 5_0003]],
                  "attention_mask": [[1, 1, 1, 1]],
                  # java
                  "forced_bos_token_id": 5_0001,
              }     ,  )



"""simple docstring"""




from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase__      :    str    							= logging.get_logger(__name__)

UpperCAmelCase__      :    Optional[int]    							= {
    'hustvl/yolos-small': 'https://huggingface.co/hustvl/yolos-small/resolve/main/config.json',
    # See all YOLOS models at https://huggingface.co/models?filter=yolos
}



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						int					       =   '''yolos'''



				def __init__(self   ,       SCREAMING_SNAKE_CASE__=7_68   ,       SCREAMING_SNAKE_CASE__=12   ,       SCREAMING_SNAKE_CASE__=12   ,       SCREAMING_SNAKE_CASE__=30_72   ,       SCREAMING_SNAKE_CASE__="gelu"   ,       SCREAMING_SNAKE_CASE__=0.0   ,       SCREAMING_SNAKE_CASE__=0.0   ,       SCREAMING_SNAKE_CASE__=0.02   ,       SCREAMING_SNAKE_CASE__=1E-12   ,       SCREAMING_SNAKE_CASE__=[5_12, 8_64]   ,       SCREAMING_SNAKE_CASE__=16   ,       SCREAMING_SNAKE_CASE__=3   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=1_00   ,       SCREAMING_SNAKE_CASE__=True   ,       SCREAMING_SNAKE_CASE__=False   ,       SCREAMING_SNAKE_CASE__=1   ,       SCREAMING_SNAKE_CASE__=5   ,       SCREAMING_SNAKE_CASE__=2   ,       SCREAMING_SNAKE_CASE__=5   ,       SCREAMING_SNAKE_CASE__=2   ,       SCREAMING_SNAKE_CASE__=0.1   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		Optional[Any]:



											"""simple docstring"""




											super().__init__(**SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   hidden_size
											SCREAMING_SNAKE_CASE__						:       int           =   num_hidden_layers
											SCREAMING_SNAKE_CASE__						:       str           =   num_attention_heads
											SCREAMING_SNAKE_CASE__						:       List[str]           =   intermediate_size
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   hidden_act
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   hidden_dropout_prob
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   initializer_range
											SCREAMING_SNAKE_CASE__						:       Dict           =   layer_norm_eps
											SCREAMING_SNAKE_CASE__						:       List[str]           =   image_size
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   patch_size
											SCREAMING_SNAKE_CASE__						:       List[str]           =   num_channels
											SCREAMING_SNAKE_CASE__						:       List[str]           =   qkv_bias
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   num_detection_tokens
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   use_mid_position_embeddings
											SCREAMING_SNAKE_CASE__						:       List[str]           =   auxiliary_loss
											# Hungarian matcher
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   class_cost
											SCREAMING_SNAKE_CASE__						:       List[str]           =   bbox_cost
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   giou_cost
											# Loss coefficients
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   bbox_loss_coefficient
											SCREAMING_SNAKE_CASE__						:       List[str]           =   giou_loss_coefficient
											SCREAMING_SNAKE_CASE__						:       int           =   eos_coefficient






class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Dict					       =   version.parse('''1.11'''  )



				@property
				def 	__magic_name__    (self    )					->		Mapping[str, Mapping[int, str]]:



											"""simple docstring"""




											return OrderedDict(
											    [
											        ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
											    ]    )



				@property
				def 	__magic_name__    (self    )					->		float:



											"""simple docstring"""




											return 1E-4



				@property
				def 	__magic_name__    (self    )					->		int:



											"""simple docstring"""




											return 12




