




"""simple docstring"""

import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a					=			get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
       from transformers.models.plbart.modeling_plbart import shift_tokens_right

a					=			50_003
a					=			50_002





@require_sentencepiece
@require_tokenizers
class 				lowercase_       (     _lowerCamelCase			,	unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase    :     Optional[Any]        =	PLBartTokenizer
 UpperCAmelCase    :     str        =	None
 UpperCAmelCase    :     Optional[int]        =	False

 def 			lowerCAmelCase_							( self       :       Tuple		):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _A					   =   PLBartTokenizer(lowerCAmelCase_    ,							language_codes='base'    ,							keep_accents=lowerCAmelCase_		)
       tokenizer.save_pretrained(self.tmpdirname		)

 def 			lowerCAmelCase_							( self       :       Any		):
       _A					   =   PLBartTokenizer(lowerCAmelCase_    ,							language_codes='base'    ,							keep_accents=lowerCAmelCase_		)

       _A					   =   tokenizer.tokenize('This is a test'		)
       self.assertListEqual(lowerCAmelCase_    ,							['▁This', '▁is', '▁a', '▁t', 'est']		)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)    ,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,							)

       _A					   =   tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]    ,							)
       _A					   =   tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]    ,							)

       _A					   =   tokenizer.convert_ids_to_tokens(lowerCAmelCase_		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]    ,							)

       _A					   =   tokenizer.vocab_size
       _A					   =   [tokenizer.convert_ids_to_tokens(lowerCAmelCase_		) for x in range(end - 4    ,							lowerCAmelCase_		)]

       self.assertListEqual(lowerCAmelCase_    ,							['__java__', '__python__', '__en_XX__', '<mask>']		)

       _A					   =   'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
       _A					   =   tokenizer(lowerCAmelCase_		).input_ids
       self.assertEqual(
           tokenizer.decode(lowerCAmelCase_    ,							skip_special_tokens=lowerCAmelCase_    ,							clean_up_tokenization_spaces=lowerCAmelCase_		)    ,							lowerCAmelCase_    ,							)



 def 			lowerCAmelCase_							( self       :       Tuple		):
       _A					   =   PLBartTokenizer(lowerCAmelCase_    ,							language_codes='multi'    ,							keep_accents=lowerCAmelCase_		)

       _A					   =   tokenizer.tokenize('This is a test'		)
       self.assertListEqual(lowerCAmelCase_    ,							['▁This', '▁is', '▁a', '▁t', 'est']		)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)    ,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,							)

       _A					   =   tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]    ,							)
       _A					   =   tokenizer.convert_tokens_to_ids(lowerCAmelCase_		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]    ,							)

       _A					   =   tokenizer.convert_ids_to_tokens(lowerCAmelCase_		)
       self.assertListEqual(
           lowerCAmelCase_    ,							[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]    ,							)
       _A					   =   tokenizer.vocab_size
       _A					   =   [tokenizer.convert_ids_to_tokens(lowerCAmelCase_		) for x in range(end - 7    ,							lowerCAmelCase_		)]

       self.assertListEqual(
           lowerCAmelCase_    ,							['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']		)
       _A					   =   'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
       _A					   =   tokenizer(lowerCAmelCase_		).input_ids
       self.assertEqual(
           tokenizer.decode(lowerCAmelCase_    ,							skip_special_tokens=lowerCAmelCase_    ,							clean_up_tokenization_spaces=lowerCAmelCase_		)    ,							lowerCAmelCase_    ,							)





@require_torch
@require_sentencepiece
@require_tokenizers
class 				lowercase_       (     unittest.TestCase							):



 '''simple docstring'''
 UpperCAmelCase    :     Any        =	'''uclanlp/plbart-python-en_XX'''
 UpperCAmelCase    :     Dict        =	[
     '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
     '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
 ]
 UpperCAmelCase    :     Dict        =	[
     '''Returns the maximum value of a b c.''',
     '''Sums the values of a b c.''',
 ]
 UpperCAmelCase    :     Any        =	[
     134,
     5452,
     3_3460,
     3_3441,
     3_3463,
     3_3465,
     3_3463,
     3_3449,
     988,
     20,
     3_3456,
     19,
     3_3456,
     771,
     39,
     4258,
     889,
     3318,
     3_3441,
     3_3463,
     3_3465,
     3_3463,
     3_3449,
     2471,
     2,
     PYTHON_CODE,
 ]

 @classmethod
 def 			lowerCAmelCase_							( cls       :       Union[str, Any]		):
       _A					   =   PLBartTokenizer.from_pretrained(
           cls.checkpoint_name    ,							language_codes='base'    ,							src_lang='python'    ,							tgt_lang='en_XX'		)
       _A					   =   1
       return cls

 def 			lowerCAmelCase_							( self       :       Any		):
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']    ,							50_001		)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']    ,							50_002		)
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']    ,							50_003		)

 def 			lowerCAmelCase_							( self       :       int		):
       _A					   =   self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
       self.assertListEqual(self.expected_src_tokens    ,							lowerCAmelCase_		)

 def 			lowerCAmelCase_							( self       :       str		):
       self.assertIn(lowerCAmelCase_    ,							self.tokenizer.all_special_ids		)
       _A					   =   [EN_CODE, 9_037, 33_442, 57, 752, 153, 14, 56, 18, 9, 2]
       _A					   =   self.tokenizer.decode(lowerCAmelCase_    ,							skip_special_tokens=lowerCAmelCase_		)
       _A					   =   self.tokenizer.decode(generated_ids[1:]    ,							skip_special_tokens=lowerCAmelCase_		)
       self.assertEqual(lowerCAmelCase_    ,							lowerCAmelCase_		)
       self.assertNotIn(self.tokenizer.eos_token    ,							lowerCAmelCase_		)

 def 			lowerCAmelCase_							( self       :       int		):
       _A					   =   ['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
       self.assertIsInstance(src_text[0]    ,							lowerCAmelCase_		)
       _A					   =   10
       _A					   =   self.tokenizer(lowerCAmelCase_    ,							max_length=lowerCAmelCase_    ,							truncation=lowerCAmelCase_		).input_ids[0]
       self.assertEqual(ids[-2]    ,							2		)
       self.assertEqual(ids[-1]    ,							lowerCAmelCase_		)
       self.assertEqual(len(lowerCAmelCase_		)    ,							lowerCAmelCase_		)

 def 			lowerCAmelCase_							( self       :       int		):
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']		)    ,							[50_004, 50_001]		)

 def 			lowerCAmelCase_							( self       :       str		):
       _A					   =   tempfile.mkdtemp()
       _A					   =   self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(lowerCAmelCase_		)
       _A					   =   PLBartTokenizer.from_pretrained(lowerCAmelCase_		)
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,							lowerCAmelCase_		)

 @require_torch
 def 			lowerCAmelCase_							( self       :       List[str]		):
       _A					   =   self.tokenizer(self.src_text    ,							text_target=self.tgt_text    ,							padding=lowerCAmelCase_    ,							return_tensors='pt'		)
       _A					   =   shift_tokens_right(batch['labels']    ,							self.tokenizer.pad_token_id		)

       # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
       self.assertEqual(batch.input_ids[1][-2:].tolist()    ,							[2, PYTHON_CODE]		)
       self.assertEqual(batch.decoder_input_ids[1][0]    ,							lowerCAmelCase_		)
       self.assertEqual(batch.decoder_input_ids[1][-1]    ,							2		)
       self.assertEqual(batch.labels[1][-2:].tolist()    ,							[2, EN_CODE]		)

 @require_torch
 def 			lowerCAmelCase_							( self       :       List[str]		):
       _A					   =   self.tokenizer(
           self.src_text    ,							text_target=self.tgt_text    ,							padding=lowerCAmelCase_    ,							truncation=lowerCAmelCase_    ,							max_length=len(self.expected_src_tokens		)    ,							return_tensors='pt'    ,							)
       _A					   =   shift_tokens_right(batch['labels']    ,							self.tokenizer.pad_token_id		)

       self.assertIsInstance(lowerCAmelCase_    ,							lowerCAmelCase_		)

       self.assertEqual((2, 26)    ,							batch.input_ids.shape		)
       self.assertEqual((2, 26)    ,							batch.attention_mask.shape		)
       _A					   =   batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens    ,							lowerCAmelCase_		)
       self.assertEqual(2    ,							batch.decoder_input_ids[0, -1]		)  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens    ,							[]		)
       self.assertEqual(self.tokenizer.suffix_tokens    ,							[self.tokenizer.eos_token_id, PYTHON_CODE]		)

 def 			lowerCAmelCase_							( self       :       Any		):
       _A					   =   self.tokenizer(self.src_text    ,							padding=lowerCAmelCase_    ,							truncation=lowerCAmelCase_    ,							max_length=3    ,							return_tensors='pt'		)
       _A					   =   self.tokenizer(
           text_target=self.tgt_text    ,							padding=lowerCAmelCase_    ,							truncation=lowerCAmelCase_    ,							max_length=10    ,							return_tensors='pt'		)
       _A					   =   targets['input_ids']
       _A					   =   shift_tokens_right(lowerCAmelCase_    ,							self.tokenizer.pad_token_id		)

       self.assertEqual(batch.input_ids.shape[1]    ,							3		)
       self.assertEqual(batch.decoder_input_ids.shape[1]    ,							10		)



 @require_torch
 def 			lowerCAmelCase_							( self       :       Any		):
       _A					   =   self.tokenizer._build_translation_inputs(
           'A test'    ,							return_tensors='pt'    ,							src_lang='en_XX'    ,							tgt_lang='java'		)

       self.assertEqual(
           nested_simplify(lowerCAmelCase_		)    ,							{
               # A, test, EOS, en_XX
               'input_ids': [[150, 242, 2, 50_003]],
               'attention_mask': [[1, 1, 1, 1]],
               # java
               'forced_bos_token_id': 50_001,
           }    ,							)



'''simple docstring'''







import subprocess
import sys

from transformers import BertConfig, BertModel, BertTokenizer, pipeline
from transformers.testing_utils import TestCasePlus, require_torch


class 						__UpperCAmelCase      (					_lowerCamelCase						):


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise RuntimeError("Offline mode is enabled, we shouldn\'t access internet")\nsocket.socket = offline_socket\n        '

								# Force fetching the files so that we can use the cache
								_snake_case   						=		'hf-internal-testing/tiny-random-bert'
								BertConfig.from_pretrained(lowerCAmelCase_  )
								BertModel.from_pretrained(lowerCAmelCase_  )
								BertTokenizer.from_pretrained(lowerCAmelCase_  )
								pipeline(task='fill-mask'    ,   model=lowerCAmelCase_  )

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run, mock]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer, pipeline\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nBertTokenizer.from_pretrained(mname)\npipe = pipeline(task="fill-mask", model=mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Faking flaky internet")\nsocket.socket = offline_socket\n        '

								# Force fetching the files so that we can use the cache
								_snake_case   						=		'hf-internal-testing/tiny-random-bert'
								BertConfig.from_pretrained(lowerCAmelCase_  )
								BertModel.from_pretrained(lowerCAmelCase_  )
								BertTokenizer.from_pretrained(lowerCAmelCase_  )
								pipeline(task='fill-mask'    ,   model=lowerCAmelCase_  )

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run, mock]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import BertConfig, BertModel, BertTokenizer\n        '

								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert-sharded"\nBertConfig.from_pretrained(mname)\nBertModel.from_pretrained(mname)\nprint("success")\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise ValueError("Offline mode is enabled")\nsocket.socket = offline_socket\n        '

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

								# next emulate no network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, mock, run]  )]

								# Doesn't fail anymore since the model is in the cache due to other tests, so commenting this.
								# env["TRANSFORMERS_OFFLINE"] = "0"
								# result = subprocess.run(cmd, env=env, check=False, capture_output=True)
								# self.assertEqual(result.returncode, 1, result.stderr)

								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import pipeline\n        '
								_snake_case   						=		'\nmname = "hf-internal-testing/tiny-random-bert"\npipe = pipeline(model=mname)\n        '

								_snake_case   						=		'\nimport socket\ndef offline_socket(*args, **kwargs): raise socket.error("Offline mode is enabled")\nsocket.socket = offline_socket\n        '
								_snake_case   						=		self.get_env()
								_snake_case   						=		'1'
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, mock, run]  )]
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   1    ,   result.stderr  )
								self.assertIn(
								    'You cannot infer task automatically within `pipeline` when using offline mode'    ,   result.stderr.decode().replace('\n'    ,   ''  )    ,   )




							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'\nfrom transformers import AutoModel\n        '
								_snake_case   						=		'\nmname = "hf-internal-testing/test_dynamic_model"\nAutoModel.from_pretrained(mname, trust_remote_code=True)\nprint("success")\n        '

								# baseline - just load from_pretrained with normal network
								_snake_case   						=		[sys.executable, '-c', '\n'.join([load, run]  )]

								# should succeed
								_snake_case   						=		self.get_env()
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

								# should succeed as TRANSFORMERS_OFFLINE=1 tells it to use local files
								_snake_case   						=		'1'
								_snake_case   						=		subprocess.run(lowerCAmelCase_    ,   env=lowerCAmelCase_    ,   check=lowerCAmelCase_    ,   capture_output=lowerCAmelCase_  )
								self.assertEqual(result.returncode    ,   0    ,   result.stderr  )
								self.assertIn('success'    ,   result.stdout.decode()  )

