






"""simple docstring"""

import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment

def 		__lowerCamelCase	(  )						->    Dict:

		"""simple docstring"""







		lowerCAmelCase_   :							List[str]									=			torch.nn.Linear(2      ,       4       )
		lowerCAmelCase_   :							Optional[Any]									=			torch.optim.AdamW(model.parameters()      ,       lr=1.0       )
		lowerCAmelCase_   :							Optional[Any]									=			torch.optim.lr_scheduler.OneCycleLR(_lowerCamelCase      ,       max_lr=0.01      ,       steps_per_epoch=2      ,       epochs=1       )
		lowerCAmelCase_   :							List[Any]									=			DataLoader(TensorDataset(torch.tensor([1, 2, 3]       )       )       )
		lowerCAmelCase_   :							Optional[int]									=			DataLoader(TensorDataset(torch.tensor([4, 5, 6]       )       )       )

		return model, optimizer, scheduler, train_dl, valid_dl

def 		__lowerCamelCase	(  __UpperCamelCase       )						->    Any:

		"""simple docstring"""







		return (model.weight.abs().sum() + model.bias.abs().sum()).item()

def 		__lowerCamelCase	(  __UpperCamelCase       )						->    Tuple:

		"""simple docstring"""







		lowerCAmelCase_   :							Tuple									=			torch.nn.Linear(*tuple(model.weight.T.shape       )       ).state_dict()
		model.load_state_dict(_lowerCamelCase       )



class 		__lowerCamelCase				( A__			):





		'''simple docstring'''






		@require_cuda
		def    lowerCamelCase       (   self							:	Union[str, Any]	):
				lowerCAmelCase_   :							str									=			Accelerator()
				assert PartialState._shared_state["_cpu"] is False
				assert PartialState._shared_state["device"].type == "cuda"
				with self.assertRaises(__a	):
						lowerCAmelCase_   :							List[str]									=			Accelerator(cpu=__a	)





		def    lowerCamelCase       (   self							:	Optional[Any]	):
				lowerCAmelCase_   :							int									=			Accelerator()
				lowerCAmelCase_   :							Any									=			GradientState()
				assert state.num_steps == 1
				lowerCAmelCase_   :							Any									=			4
				assert state.num_steps == 4

				assert state.sync_gradients is True
				lowerCAmelCase_   :							Any									=			False
				assert state.sync_gradients is False
				GradientState._reset_state()





		def    lowerCamelCase       (   self							:	Tuple	):
				lowerCAmelCase_   :							Optional[Any]									=			Accelerator()
				lowerCAmelCase_   :							Any									=			create_components()

				(
				    lowerCAmelCase_    
				)   :							Union[str, Any]									=			accelerator.prepare(__a						,			__a						,			__a						,			__a						,			__a	)

				self.assertTrue(prepared_model in accelerator._models	)
				self.assertTrue(prepared_optimizer in accelerator._optimizers	)
				self.assertTrue(prepared_scheduler in accelerator._schedulers	)
				self.assertTrue(prepared_train_dl in accelerator._dataloaders	)
				self.assertTrue(prepared_valid_dl in accelerator._dataloaders	)





		def    lowerCamelCase       (   self							:	Optional[int]	):
				lowerCAmelCase_   :							Tuple									=			Accelerator()
				lowerCAmelCase_   :							List[str]									=			create_components()
				accelerator.prepare(__a						,			__a						,			__a						,			__a						,			__a	)
				accelerator.free_memory()

				self.assertTrue(len(accelerator._models	) == 0	)
				self.assertTrue(len(accelerator._optimizers	) == 0	)
				self.assertTrue(len(accelerator._schedulers	) == 0	)
				self.assertTrue(len(accelerator._dataloaders	) == 0	)





		def    lowerCamelCase       (   self							:	Any	):
				PartialState._reset_state()

				# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
				def noop(*a_							:	Optional[Any]						,			**a_							:	Tuple	):
						pass

				with patch("torch.cuda.set_device"						,			__a	), patch_environment(ACCELERATE_TORCH_DEVICE="cuda:64"	):
						lowerCAmelCase_   :							int									=			Accelerator()
						self.assertEqual(str(accelerator.state.device	)						,			"cuda:64"	)





		def    lowerCamelCase       (   self							:	List[str]	):
				lowerCAmelCase_   :							int									=			Accelerator()
				lowerCAmelCase_   :							List[Any]									=			create_components()
				accelerator.prepare(__a						,			__a						,			__a						,			__a						,			__a	)

				lowerCAmelCase_   :							Optional[Any]									=			get_signature(__a	)

				with tempfile.TemporaryDirectory() as tmpdirname:
						accelerator.save_state(__a	)

						# make sure random weights don't match
						load_random_weights(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) > 1e-3	)

						# make sure loaded weights match
						accelerator.load_state(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) < 1e-3	)





		def    lowerCamelCase       (   self							:	Union[str, Any]	):
				lowerCAmelCase_   :							Union[str, Any]									=			Accelerator()
				lowerCAmelCase_   :							List[Any]									=			create_components()
				accelerator.prepare(__a						,			__a						,			__a						,			__a						,			__a	)

				lowerCAmelCase_   :							Optional[Any]									=			get_signature(__a	)

				# saving hook
				def save_config(a_							:	List[str]						,			a_							:	Optional[int]						,			a_							:	Dict	):
						lowerCAmelCase_   :							Optional[int]									=			{"class_name": models[0].__class__.__name__}

						with open(os.path.join(__a						,			"data.json"	)						,			"w"	) as f:
								json.dump(__a						,			__a	)

        # loading hook
				def load_config(a_							:	List[str]						,			a_							:	Tuple	):
						with open(os.path.join(__a						,			"data.json"	)						,			"r"	) as f:
								lowerCAmelCase_   :							Optional[int]									=			json.load(__a	)

						lowerCAmelCase_   :							List[Any]									=			config["class_name"]

				lowerCAmelCase_   :							Optional[int]									=			accelerator.register_save_state_pre_hook(__a	)
				lowerCAmelCase_   :							List[str]									=			accelerator.register_load_state_pre_hook(__a	)

				with tempfile.TemporaryDirectory() as tmpdirname:
						accelerator.save_state(__a	)

						# make sure random weights don't match with hooks
						load_random_weights(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) > 1e-3	)

						# random class name to verify correct one is loaded
						lowerCAmelCase_   :							Any									=			"random"

						# make sure loaded weights match with hooks
						accelerator.load_state(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) < 1e-3	)

						# mode.class_name is loaded from config
						self.assertTrue(model.class_name == model.__class__.__name__	)

				# remove hooks
				save_hook.remove()
				load_hook.remove()

				with tempfile.TemporaryDirectory() as tmpdirname:
						accelerator.save_state(__a	)

						# make sure random weights don't match with hooks removed
						load_random_weights(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) > 1e-3	)

						# random class name to verify correct one is loaded
						lowerCAmelCase_   :							int									=			"random"

						# make sure loaded weights match with hooks removed
						accelerator.load_state(__a	)
						self.assertTrue(abs(model_signature - get_signature(__a	)	) < 1e-3	)

						# mode.class_name is NOT loaded from config
						self.assertTrue(model.class_name != model.__class__.__name__	)





		def    lowerCamelCase       (   self							:	Tuple	):
				lowerCAmelCase_   :							Any									=			Accelerator()
				lowerCAmelCase_   :							List[str]									=			create_components()
				lowerCAmelCase_   :							Any									=			None

				# This should work
				lowerCAmelCase_   :							Tuple									=			accelerator.prepare(
				    __a						,			__a						,			__a						,			__a						,			__a						,			__a	)
				self.assertTrue(dummy_obj is None	)





		def    lowerCamelCase       (   self							:	Optional[int]	):
				lowerCAmelCase_   :							Optional[Any]									=			Accelerator()
				lowerCAmelCase_   :							Any									=			create_components()
				lowerCAmelCase_   :							Any									=			[1, 2, 3]

				# This should work
				lowerCAmelCase_   :							Dict									=			accelerator.prepare(
				    __a						,			__a						,			__a						,			__a						,			__a						,			__a	)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Dummy object should have `_is_accelerate_prepared` set to `True`"						,			)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Model is missing `_is_accelerator_prepared` or is set to `False`"						,			)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Optimizer is missing `_is_accelerator_prepared` or is set to `False`"						,			)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Scheduler is missing `_is_accelerator_prepared` or is set to `False`"						,			)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`"						,			)
				self.assertEqual(
				    getattr(__a						,			"_is_accelerate_prepared"						,			__a	)						,			__a						,			"Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`"						,			)





		@slow
		@require_bnb
		def    lowerCamelCase       (   self							:	Optional[int]	):
				from transformers import AutoModelForCausalLM

				lowerCAmelCase_   :							List[str]									=			AutoModelForCausalLM.from_pretrained(
				    "EleutherAI/gpt-neo-125m"						,			load_in_abit=__a						,			device_map={"": 0}						,			)
				lowerCAmelCase_   :							Dict									=			Accelerator()

				# This should work
				lowerCAmelCase_   :							List[str]									=			accelerator.prepare(__a	)





		@slow
		@require_bnb
		def    lowerCamelCase       (   self							:	Tuple	):
				from transformers import AutoModelForCausalLM

				lowerCAmelCase_   :							List[Any]									=			Accelerator()

				with init_empty_weights():
						lowerCAmelCase_   :							Tuple									=			AutoModelForCausalLM.from_pretrained(
						    "EleutherAI/gpt-neo-125m"						,			)
						model.tie_weights()
						lowerCAmelCase_   :							Optional[int]									=			infer_auto_device_map(__a	)
						lowerCAmelCase_   :							Dict									=			"cpu"

				lowerCAmelCase_   :							Tuple									=			AutoModelForCausalLM.from_pretrained(
				    "EleutherAI/gpt-neo-125m"						,			device_map=__a						,			load_in_abit=__a						,			llm_inta_enable_fpaa_cpu_offload=__a	)

				# This should not work and get value error
				with self.assertRaises(__a	):
						lowerCAmelCase_   :							Optional[Any]									=			accelerator.prepare(__a	)





		@slow
		@require_bnb
		@require_multi_gpu
		def    lowerCamelCase       (   self							:	Optional[int]	):
				from transformers import AutoModelForCausalLM

				lowerCAmelCase_   :							Union[str, Any]									=			{"distributed_type": DistributedType.MULTI_GPU}

				with init_empty_weights():
						lowerCAmelCase_   :							List[Any]									=			AutoModelForCausalLM.from_pretrained(
						    "EleutherAI/gpt-neo-125m"						,			)
						model.tie_weights()
						lowerCAmelCase_   :							Optional[Any]									=			infer_auto_device_map(__a	)
						lowerCAmelCase_   :							Optional[int]									=			1

				lowerCAmelCase_   :							int									=			AutoModelForCausalLM.from_pretrained(
				    "EleutherAI/gpt-neo-125m"						,			load_in_abit=__a						,			device_map=__a						,			)
				lowerCAmelCase_   :							str									=			Accelerator()

				# This should not work and get value error
				with self.assertRaises(__a	):
						lowerCAmelCase_   :							List[Any]									=			accelerator.prepare(__a	)

				PartialState._reset_state()





		@slow
		@require_bnb
		@require_multi_gpu
		def    lowerCamelCase       (   self							:	List[str]	):
				from transformers import AutoModelForCausalLM

				with init_empty_weights():
						lowerCAmelCase_   :							List[str]									=			AutoModelForCausalLM.from_pretrained(
						    "EleutherAI/gpt-neo-125m"						,			)
						lowerCAmelCase_   :							List[Any]									=			infer_auto_device_map(__a	)
						lowerCAmelCase_   :							List[str]									=			1

				lowerCAmelCase_   :							Tuple									=			AutoModelForCausalLM.from_pretrained(
				    "EleutherAI/gpt-neo-125m"						,			load_in_abit=__a						,			device_map=__a						,			)
				lowerCAmelCase_   :							Dict									=			Accelerator()

				# This should work
				lowerCAmelCase_   :							Union[str, Any]									=			accelerator.prepare(__a	)





		@require_cuda
		def    lowerCamelCase       (   self							:	Optional[int]	):
				lowerCAmelCase_   :							str									=			torch.nn.Linear(10						,			10	)
				lowerCAmelCase_   :							Union[str, Any]									=			torch.optim.SGD(model.parameters()						,			lr=0.01	)
				lowerCAmelCase_   :							Dict									=			Accelerator(cpu=__a	)
				lowerCAmelCase_   :							Optional[int]									=			accelerator.prepare(__a	)





import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    LiltForQuestionAnswering,
						    LiltForSequenceClassification,
						    LiltForTokenClassification,
						    LiltModel,
						)
						from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST



class   UpperCAmelCase_    :







							def __init__(   self,			__a,			__a=13,			__a=7,			__a=True,			__a=True,			__a=True,			__a=True,			__a=99,			__a=24,			__a=2,			__a=6,			__a=37,			__a="gelu",			__a=0.1,			__a=0.1,			__a=512,			__a=16,			__a=2,			__a=0.02,			__a=3,			__a=None,			__a=1000,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    parent
														_lowerCAmelCase     :							List[str]    =    batch_size
														_lowerCAmelCase     :							int    =    seq_length
														_lowerCAmelCase     :							Optional[int]    =    is_training
														_lowerCAmelCase     :							Dict    =    use_input_mask
														_lowerCAmelCase     :							List[str]    =    use_token_type_ids
														_lowerCAmelCase     :							str    =    use_labels
														_lowerCAmelCase     :							Optional[Any]    =    vocab_size
														_lowerCAmelCase     :							Tuple    =    hidden_size
														_lowerCAmelCase     :							List[Any]    =    num_hidden_layers
														_lowerCAmelCase     :							Optional[Any]    =    num_attention_heads
														_lowerCAmelCase     :							Any    =    intermediate_size
														_lowerCAmelCase     :							List[str]    =    hidden_act
														_lowerCAmelCase     :							Union[str, Any]    =    hidden_dropout_prob
														_lowerCAmelCase     :							Any    =    attention_probs_dropout_prob
														_lowerCAmelCase     :							int    =    max_position_embeddings
														_lowerCAmelCase     :							Optional[int]    =    type_vocab_size
														_lowerCAmelCase     :							Optional[Any]    =    type_sequence_label_size
														_lowerCAmelCase     :							List[str]    =    initializer_range
														_lowerCAmelCase     :							List[Any]    =    num_labels
														_lowerCAmelCase     :							Tuple    =    scope
														_lowerCAmelCase     :							str    =    range_bbox







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    ids_tensor([self.batch_size, self.seq_length],			self.vocab_size)

														_lowerCAmelCase     :							int    =    ids_tensor([self.batch_size, self.seq_length, 4],			self.range_bbox)
														# Ensure that bbox is legal
														for i in range(bbox.shape[0]):
																					for j in range(bbox.shape[1]):
																												if bbox[i, j, 3] < bbox[i, j, 1]:
																																			_lowerCAmelCase     :							Dict    =    bbox[i, j, 3]
																																			_lowerCAmelCase     :							int    =    bbox[i, j, 1]
																																			_lowerCAmelCase     :							Tuple    =    t
																												if bbox[i, j, 2] < bbox[i, j, 0]:
																																			_lowerCAmelCase     :							str    =    bbox[i, j, 2]
																																			_lowerCAmelCase     :							List[Any]    =    bbox[i, j, 0]
																																			_lowerCAmelCase     :							str    =    t

														_lowerCAmelCase     :							Optional[Any]    =    None
														if self.use_input_mask:
																					_lowerCAmelCase     :							Union[str, Any]    =    ids_tensor([self.batch_size, self.seq_length],			vocab_size=2)

														_lowerCAmelCase     :							Dict    =    None
														if self.use_token_type_ids:
																					_lowerCAmelCase     :							Any    =    ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size)

														_lowerCAmelCase     :							Optional[int]    =    None
														_lowerCAmelCase     :							Optional[Any]    =    None
														if self.use_labels:
																					_lowerCAmelCase     :							Optional[Any]    =    ids_tensor([self.batch_size],			self.type_sequence_label_size)
																					_lowerCAmelCase     :							Any    =    ids_tensor([self.batch_size, self.seq_length],			self.num_labels)

														_lowerCAmelCase     :							Optional[int]    =    self.get_config()

														return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels







							def    snake_case__      (   self):

														'''simple docstring'''

														return LiltConfig(
														    vocab_size=self.vocab_size,			hidden_size=self.hidden_size,			num_hidden_layers=self.num_hidden_layers,			num_attention_heads=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			max_position_embeddings=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			initializer_range=self.initializer_range,			)







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    LiltModel(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Dict    =    model(__a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a)
														_lowerCAmelCase     :							str    =    model(__a,			bbox=__a,			token_type_ids=__a)
														_lowerCAmelCase     :							List[Any]    =    model(__a,			bbox=__a)

														self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size))
														self.parent.assertEqual(result.pooler_output.shape,			(self.batch_size, self.hidden_size))







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    self.num_labels
														_lowerCAmelCase     :							Optional[Any]    =    LiltForTokenClassification(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Dict    =    model(
														    __a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a,			labels=__a)
														self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.num_labels))







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[int]    =    LiltForQuestionAnswering(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Tuple    =    model(
														    __a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a,			start_positions=__a,			end_positions=__a,			)
														self.parent.assertEqual(result.start_logits.shape,			(self.batch_size, self.seq_length))
														self.parent.assertEqual(result.end_logits.shape,			(self.batch_size, self.seq_length))







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    self.prepare_config_and_inputs()
														(
														    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    
														)     :							Dict    =    config_and_inputs
														_lowerCAmelCase     :							List[Any]    =    {
														    "input_ids": input_ids,
														    "bbox": bbox,
														    "token_type_ids": token_type_ids,
														    "attention_mask": input_mask,
														}
														return config, inputs_dict



@require_torch
class   UpperCAmelCase_    (       a   ,					a   ,					a   ,					unittest.TestCase):
							lowerCamelCase__   			=		(
							    (
							        LiltModel,
							        LiltForSequenceClassification,
							        LiltForTokenClassification,
							        LiltForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase__   			=		(
							    {
							        'feature-extraction': LiltModel,
							        'question-answering': LiltForQuestionAnswering,
							        'text-classification': LiltForSequenceClassification,
							        'token-classification': LiltForTokenClassification,
							        'zero-shot': LiltForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase__   			=		False
							lowerCamelCase__   			=		False







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a):

														'''simple docstring'''

														return True







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    LiltModelTester(self)
														_lowerCAmelCase     :							Union[str, Any]    =    ConfigTester(self,			config_class=__a,			hidden_size=37)







							def    snake_case__      (   self):

														'''simple docstring'''

														self.config_tester.run_common_tests()







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    self.model_tester.prepare_config_and_inputs()
														for type in ["absolute", "relative_key", "relative_key_query"]:
																					_lowerCAmelCase     :							Any    =    type
																					self.model_tester.create_and_check_model(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[int]    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*__a)







							@slow
							def    snake_case__      (   self):

														'''simple docstring'''

														for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					_lowerCAmelCase     :							str    =    LiltModel.from_pretrained(__a)
																					self.assertIsNotNone(__a)



@require_torch
@slow
class   UpperCAmelCase_    (       unittest.TestCase):







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base").to(__a)

														_lowerCAmelCase     :							Any    =    torch.tensor([[1, 2]],			device=__a)
														_lowerCAmelCase     :							str    =    torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]],			device=__a)

														# forward pass
														with torch.no_grad():
																					_lowerCAmelCase     :							Optional[Any]    =    model(input_ids=__a,			bbox=__a)

														_lowerCAmelCase     :							Optional[int]    =    torch.Size([1, 2, 768])
														_lowerCAmelCase     :							List[str]    =    torch.tensor(
														    [[-0.0_653, 0.0_950, -0.0_061], [-0.0_545, 0.0_926, -0.0_324]],			device=__a,			)

														self.assertTrue(outputs.last_hidden_state.shape,			__a)
														self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3],			__a,			atol=1E-3))
