





"""simple docstring"""





import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 				_lowerCamelCase					(				__UpperCAmelCase     ,	unittest.TestCase       ):
						UpperCAmelCase_       =	LEDTokenizer
						UpperCAmelCase_       =	LEDTokenizerFast
						UpperCAmelCase_       =	True


						def 			snake_case_      (self						)       ->      Optional[Any]:
								super().setUp()
								UpperCamelCase      =		[
								    "l",
								    "o",
								    "w",
								    "e",
								    "r",
								    "s",
								    "t",
								    "i",
								    "d",
								    "n",
								    "\u0120",
								    "\u0120l",
								    "\u0120n",
								    "\u0120lo",
								    "\u0120low",
								    "er",
								    "\u0120lowest",
								    "\u0120newer",
								    "\u0120wider",
								    "<unk>",
								]
								UpperCamelCase      =		dict(zip(__SCREAMING_SNAKE_CASE      ,							range(len(__SCREAMING_SNAKE_CASE						)						)						)						)
								UpperCamelCase      =		["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
								UpperCamelCase      =		{"unk_token": "<unk>"}

								UpperCamelCase      =		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]						)
								UpperCamelCase      =		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["merges_file"]						)
								with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"						) as fp:
										fp.write(json.dumps(__SCREAMING_SNAKE_CASE						) + "\n"						)
								with open(self.merges_file      ,							"w"      ,							encoding="utf-8"						) as fp:
										fp.write("\n".join(__SCREAMING_SNAKE_CASE						)						)


						def 			snake_case_      (self      ,							**__a						)       ->      Any:
								kwargs.update(self.special_tokens_map						)
								return self.tokenizer_class.from_pretrained(self.tmpdirname      ,							**__SCREAMING_SNAKE_CASE						)


						def 			snake_case_      (self      ,							**__a						)       ->      Dict:
								kwargs.update(self.special_tokens_map						)
								return self.rust_tokenizer_class.from_pretrained(self.tmpdirname      ,							**__SCREAMING_SNAKE_CASE						)


						def 			snake_case_      (self      ,							__a						)       ->      Optional[Any]:
								return "lower newer", "lower newer"


						@cached_property
						def 			snake_case_      (self						)       ->      Tuple:
								return LEDTokenizer.from_pretrained("allenai/led-base-16384"						)


						@cached_property
						def 			snake_case_      (self						)       ->      Optional[Any]:
								return LEDTokenizerFast.from_pretrained("allenai/led-base-16384"						)


						@require_torch
						def 			snake_case_      (self						)       ->      Union[str, Any]:
								UpperCamelCase      =		["A long paragraph for summarization.", "Another paragraph for summarization."]
								UpperCamelCase      =		[0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		tokenizer(__SCREAMING_SNAKE_CASE      ,							max_length=len(__SCREAMING_SNAKE_CASE						)      ,							padding=__SCREAMING_SNAKE_CASE      ,							return_tensors="pt"						)
										self.assertIsInstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE						)

										self.assertEqual((2, 9)      ,							batch.input_ids.shape						)
										self.assertEqual((2, 9)      ,							batch.attention_mask.shape						)
										UpperCamelCase      =		batch.input_ids.tolist()[0]
										self.assertListEqual(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE						)


						@require_torch
						def 			snake_case_      (self						)       ->      List[str]:
								UpperCamelCase      =		["A long paragraph for summarization.", "Another paragraph for summarization."]
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		tokenizer(__SCREAMING_SNAKE_CASE      ,							padding=__SCREAMING_SNAKE_CASE      ,							return_tensors="pt"						)
										self.assertIn("input_ids"      ,							__SCREAMING_SNAKE_CASE						)
										self.assertIn("attention_mask"      ,							__SCREAMING_SNAKE_CASE						)
										self.assertNotIn("labels"      ,							__SCREAMING_SNAKE_CASE						)
										self.assertNotIn("decoder_attention_mask"      ,							__SCREAMING_SNAKE_CASE						)


						@require_torch
						def 			snake_case_      (self						)       ->      Tuple:
								UpperCamelCase      =		[
								    "Summary of the text.",
								    "Another summary.",
								]
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		tokenizer(text_target=__SCREAMING_SNAKE_CASE      ,							max_length=32      ,							padding="max_length"      ,							return_tensors="pt"						)
										self.assertEqual(32      ,							targets["input_ids"].shape[1]						)


						@require_torch
						def 			snake_case_      (self						)       ->      Optional[int]:

								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		tokenizer(
										    ["I am a small frog" * 10_24, "I am a small frog"]      ,							padding=__SCREAMING_SNAKE_CASE      ,							truncation=__SCREAMING_SNAKE_CASE      ,							return_tensors="pt"						)
										self.assertIsInstance(__SCREAMING_SNAKE_CASE      ,							__SCREAMING_SNAKE_CASE						)
										self.assertEqual(batch.input_ids.shape      ,							(2, 51_22)						)


						@require_torch
						def 			snake_case_      (self						)       ->      Union[str, Any]:
								UpperCamelCase      =		["A long paragraph for summarization."]
								UpperCamelCase      =		[
								    "Summary of the text.",
								]
								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		tokenizer(__SCREAMING_SNAKE_CASE      ,							return_tensors="pt"						)
										UpperCamelCase      =		tokenizer(text_target=__SCREAMING_SNAKE_CASE      ,							return_tensors="pt"						)
										UpperCamelCase      =		inputs["input_ids"]
										UpperCamelCase      =		targets["input_ids"]
										self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()						)
										self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()						)
										self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()						)
										self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()						)


						@require_torch
						def 			snake_case_      (self						)       ->      int:

								for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
										UpperCamelCase      =		["Summary of the text.", "Another summary."]
										UpperCamelCase      =		[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

										UpperCamelCase      =		tokenizer(__SCREAMING_SNAKE_CASE      ,							padding=__SCREAMING_SNAKE_CASE						)
										UpperCamelCase      =		[[0] * len(__SCREAMING_SNAKE_CASE						) for x in encoded_output["input_ids"]]
										UpperCamelCase      =		tokenizer.pad(__SCREAMING_SNAKE_CASE						)
										self.assertSequenceEqual(outputs["global_attention_mask"]      ,							__SCREAMING_SNAKE_CASE						)


						def 			snake_case_      (self						)       ->      Optional[Any]:
								pass




						def 			snake_case_      (self						)       ->      int:

								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
												UpperCamelCase      =		self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE						)
												UpperCamelCase      =		self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE      ,							**__SCREAMING_SNAKE_CASE						)
												UpperCamelCase      =		"A, <mask> AllenNLP sentence."
												UpperCamelCase      =		tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE      ,							add_special_tokens=__SCREAMING_SNAKE_CASE      ,							return_token_type_ids=__SCREAMING_SNAKE_CASE						)
												UpperCamelCase      =		tokenizer_p.encode_plus(__SCREAMING_SNAKE_CASE      ,							add_special_tokens=__SCREAMING_SNAKE_CASE      ,							return_token_type_ids=__SCREAMING_SNAKE_CASE						)
												self.assertEqual(sum(tokens_r["token_type_ids"]						)      ,							sum(tokens_p["token_type_ids"]						)						)
												self.assertEqual(
												    sum(tokens_r["attention_mask"]						) / len(tokens_r["attention_mask"]						)      ,							sum(tokens_p["attention_mask"]						) / len(tokens_p["attention_mask"]						)      ,							)

												UpperCamelCase      =		tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]						)
												UpperCamelCase      =		tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]						)
												self.assertSequenceEqual(tokens_p["input_ids"]      ,							[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]						)
												self.assertSequenceEqual(tokens_r["input_ids"]      ,							[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]						)

												self.assertSequenceEqual(
												    __SCREAMING_SNAKE_CASE      ,							["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)
												self.assertSequenceEqual(
												    __SCREAMING_SNAKE_CASE      ,							["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]						)











import argparse

import torch

from transformers import LxmertConfig, LxmertForPreTraining, load_tf_weights_in_lxmert
from transformers.utils import logging


logging.set_verbosity_info()





def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase				):
    # Initialise PyTorch model
    __a						       =				LxmertConfig.from_json_file(_UpperCAmelCase				)
    print(f'Building PyTorch model from configuration: {config}'				)
    __a						       =				LxmertForPreTraining(_UpperCAmelCase				)

    # Load weights from tf checkpoint
    load_tf_weights_in_lxmert(_UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase				)

    # Save pytorch-model
    print(f'Save PyTorch model to {pytorch_dump_path}'				)
    torch.save(model.state_dict()   , _UpperCAmelCase				)


if __name__ == "__main__":
 __snake_case						:List[Any]      		=     argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
 )
 parser.add_argument(
     '''--config_file''',
     default=None,
     type=str,
     required=True,
     help='''The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.''',
 )
 parser.add_argument(
     '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
 )
 __snake_case						:Optional[Any]      		=     parser.parse_args()
 convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path)





