

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCAmelCase							:  str   	=						logging.get_logger(__name__)

lowerCAmelCase							:  Tuple   	=						{"""vocab_file""": """spiece.model"""}

lowerCAmelCase							:  Optional[Any]   	=						{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

lowerCAmelCase							:  Any   	=						{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
lowerCAmelCase							:  Any   	=						0
lowerCAmelCase							:  List[Any]   	=						1
lowerCAmelCase							:  Union[str, Any]   	=						2
lowerCAmelCase							:  Dict   	=						3
lowerCAmelCase							:  List[Any]   	=						4



class   __lowercase				( UpperCAmelCase_	):




			"""simple docstring"""







			_UpperCAmelCase   :							List[Any]										=     VOCAB_FILES_NAMES
			_UpperCAmelCase   :							List[Any]										=     PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase   :							List[str]										=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase   :							List[Any]										=     '''left'''



			def __init__( self  :     List[str]   ,						lowerCAmelCase__  :     List[Any]   ,						lowerCAmelCase__  :     Union[str, Any]=False   ,						lowerCAmelCase__  :     Dict=True   ,						lowerCAmelCase__  :     str=False   ,						lowerCAmelCase__  :     Any="<s>"   ,						lowerCAmelCase__  :     Optional[int]="</s>"   ,						lowerCAmelCase__  :     Optional[int]="<unk>"   ,						lowerCAmelCase__  :     List[str]="<sep>"   ,						lowerCAmelCase__  :     Tuple="<pad>"   ,						lowerCAmelCase__  :     int="<cls>"   ,						lowerCAmelCase__  :     List[str]="<mask>"   ,						lowerCAmelCase__  :     List[Any]=["<eop>", "<eod>"]   ,						lowerCAmelCase__  :     Optional[Dict[str, Any]] = None   ,						**lowerCAmelCase__  :     int   ,						):
									# Mask token behave like a normal word, i.e. include the space before it
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       AddedToken(lowerCAmelCase__   ,						lstrip=lowerCAmelCase__   ,						rstrip=lowerCAmelCase__) if isinstance(lowerCAmelCase__   ,						lowerCAmelCase__) else mask_token

									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=lowerCAmelCase__   ,						remove_space=lowerCAmelCase__   ,						keep_accents=lowerCAmelCase__   ,						bos_token=lowerCAmelCase__   ,						eos_token=lowerCAmelCase__   ,						unk_token=lowerCAmelCase__   ,						sep_token=lowerCAmelCase__   ,						pad_token=lowerCAmelCase__   ,						cls_token=lowerCAmelCase__   ,						mask_token=lowerCAmelCase__   ,						additional_special_tokens=lowerCAmelCase__   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**lowerCAmelCase__   ,						)

									SCREAMING_SNAKE_CASE_: Dict  		=       3

									SCREAMING_SNAKE_CASE_: List[str]  		=       do_lower_case
									SCREAMING_SNAKE_CASE_: List[Any]  		=       remove_space
									SCREAMING_SNAKE_CASE_: int  		=       keep_accents
									SCREAMING_SNAKE_CASE_: Tuple  		=       vocab_file

									SCREAMING_SNAKE_CASE_: Any  		=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(lowerCAmelCase__)



			@property
			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									return len(self.sp_model)



			def        _SCREAMING_SNAKE_CASE			( self  :     List[str]):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       {self.convert_ids_to_tokens(lowerCAmelCase__): i for i in range(self.vocab_size)}
									vocab.update(self.added_tokens_encoder)
									return vocab



			def __getstate__( self  :     Optional[int]):
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.__dict__.copy()
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       None
									return state



			def __setstate__( self  :     Tuple   ,						lowerCAmelCase__  :     Any):
									SCREAMING_SNAKE_CASE_: Any  		=       d

									# for backward compatibility
									if not hasattr(self   ,						"sp_model_kwargs"):
															SCREAMING_SNAKE_CASE_: List[Any]  		=       {}

									SCREAMING_SNAKE_CASE_: int  		=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(self.vocab_file)



			def        _SCREAMING_SNAKE_CASE			( self  :     Tuple   ,						lowerCAmelCase__  :     Any):
									if self.remove_space:
															SCREAMING_SNAKE_CASE_: int  		=       " ".join(inputs.strip().split())
									else:
															SCREAMING_SNAKE_CASE_: int  		=       inputs
									SCREAMING_SNAKE_CASE_: Tuple  		=       outputs.replace("``"   ,						"\"").replace("''"   ,						"\"")

									if not self.keep_accents:
															SCREAMING_SNAKE_CASE_: List[str]  		=       unicodedata.normalize("NFKD"   ,						lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_: int  		=       "".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__)])
									if self.do_lower_case:
															SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       outputs.lower()

									return outputs



			def        _SCREAMING_SNAKE_CASE			( self  :     str   ,						lowerCAmelCase__  :     str):
									SCREAMING_SNAKE_CASE_: str  		=       self.preprocess_text(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       self.sp_model.encode(lowerCAmelCase__   ,						out_type=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_: Tuple  		=       []
									for piece in pieces:
															if len(lowerCAmelCase__) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
																					SCREAMING_SNAKE_CASE_: List[str]  		=       self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__   ,						""))
																					if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																											if len(cur_pieces[0]) == 1:
																																	SCREAMING_SNAKE_CASE_: List[str]  		=       cur_pieces[1:]
																											else:
																																	SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       cur_pieces[0][1:]
																					cur_pieces.append(piece[-1])
																					new_pieces.extend(lowerCAmelCase__)
															else:
																					new_pieces.append(lowerCAmelCase__)

									return new_pieces



			def        _SCREAMING_SNAKE_CASE			( self  :     List[Any]   ,						lowerCAmelCase__  :     Dict):
									return self.sp_model.PieceToId(lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     int   ,						lowerCAmelCase__  :     Dict):
									return self.sp_model.IdToPiece(lowerCAmelCase__)



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict   ,						lowerCAmelCase__  :     Union[str, Any]):
									SCREAMING_SNAKE_CASE_: List[str]  		=       "".join(lowerCAmelCase__).replace(lowerCAmelCase__   ,						" ").strip()
									return out_string



			def        _SCREAMING_SNAKE_CASE			( self  :     Any   ,						lowerCAmelCase__  :     List[int]   ,						lowerCAmelCase__  :     bool = False   ,						lowerCAmelCase__  :     bool = None   ,						lowerCAmelCase__  :     bool = True   ,						**lowerCAmelCase__  :     Optional[int]   ,						):
									SCREAMING_SNAKE_CASE_: List[Any]  		=       kwargs.pop("use_source_tokenizer"   ,						lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: int  		=       self.convert_ids_to_tokens(lowerCAmelCase__   ,						skip_special_tokens=lowerCAmelCase__)

									# To avoid mixing byte-level and unicode for byte-level BPT
									# we need to build string separately for added tokens and byte-level tokens
									# cf. https://github.com/huggingface/transformers/issues/1133
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       []
									SCREAMING_SNAKE_CASE_: Optional[Any]  		=       []
									for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																					continue
															if token in self.added_tokens_encoder:
																					if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__))
																											SCREAMING_SNAKE_CASE_: Dict  		=       []
																					sub_texts.append(lowerCAmelCase__)
															else:
																					current_sub_text.append(lowerCAmelCase__)
									if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__))

									# Mimic the behavior of the Rust tokenizer:
									# By default, there are no spaces between special tokens
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       "".join(lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       (
									    clean_up_tokenization_spaces
									    if clean_up_tokenization_spaces is not None
									    else self.clean_up_tokenization_spaces
									)
									if clean_up_tokenization_spaces:
															SCREAMING_SNAKE_CASE_: Dict  		=       self.clean_up_tokenization(lowerCAmelCase__)
															return clean_text
									else:
															return text



			def        _SCREAMING_SNAKE_CASE			( self  :     int   ,						lowerCAmelCase__  :     List[int]   ,						lowerCAmelCase__  :     Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_: Optional[int]  		=       [self.sep_token_id]
									SCREAMING_SNAKE_CASE_: Union[str, Any]  		=       [self.cls_token_id]
									if token_ids_a is None:
															return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls



			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     List[int]   ,						lowerCAmelCase__  :     Optional[List[int]] = None   ,						lowerCAmelCase__  :     bool = False):

									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase__   ,						token_ids_a=lowerCAmelCase__   ,						already_has_special_tokens=lowerCAmelCase__)

									if token_ids_a is not None:
															return ([0] * len(lowerCAmelCase__)) + [1] + ([0] * len(lowerCAmelCase__)) + [1, 1]
									return ([0] * len(lowerCAmelCase__)) + [1, 1]



			def        _SCREAMING_SNAKE_CASE			( self  :     Dict   ,						lowerCAmelCase__  :     List[int]   ,						lowerCAmelCase__  :     Optional[List[int]] = None):
									SCREAMING_SNAKE_CASE_: Tuple  		=       [self.sep_token_id]
									SCREAMING_SNAKE_CASE_: List[str]  		=       [2]

									if token_ids_a is None:
															return len(token_ids_a + sep) * [0] + cls_segment_id
									return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id




			def        _SCREAMING_SNAKE_CASE			( self  :     Optional[int]   ,						lowerCAmelCase__  :     str   ,						lowerCAmelCase__  :     Optional[str] = None):
									if not os.path.isdir(lowerCAmelCase__):
															logger.error(F"Vocabulary path ({save_directory}) should be a directory")
															return
									SCREAMING_SNAKE_CASE_: int  		=       os.path.join(
									    lowerCAmelCase__   ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

									if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase__) and os.path.isfile(self.vocab_file):
															copyfile(self.vocab_file   ,						lowerCAmelCase__)
									elif not os.path.isfile(self.vocab_file):
															with open(lowerCAmelCase__   ,						"wb") as fi:
																					SCREAMING_SNAKE_CASE_: Dict  		=       self.sp_model.serialized_model_proto()
																					fi.write(lowerCAmelCase__)

									return (out_vocab_file,)











import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    LiltForQuestionAnswering,
						    LiltForSequenceClassification,
						    LiltForTokenClassification,
						    LiltModel,
						)
						from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST



class   UpperCAmelCase_    :







							def __init__(   self,			__a,			__a=13,			__a=7,			__a=True,			__a=True,			__a=True,			__a=True,			__a=99,			__a=24,			__a=2,			__a=6,			__a=37,			__a="gelu",			__a=0.1,			__a=0.1,			__a=512,			__a=16,			__a=2,			__a=0.02,			__a=3,			__a=None,			__a=1000,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    parent
														_lowerCAmelCase     :							List[str]    =    batch_size
														_lowerCAmelCase     :							int    =    seq_length
														_lowerCAmelCase     :							Optional[int]    =    is_training
														_lowerCAmelCase     :							Dict    =    use_input_mask
														_lowerCAmelCase     :							List[str]    =    use_token_type_ids
														_lowerCAmelCase     :							str    =    use_labels
														_lowerCAmelCase     :							Optional[Any]    =    vocab_size
														_lowerCAmelCase     :							Tuple    =    hidden_size
														_lowerCAmelCase     :							List[Any]    =    num_hidden_layers
														_lowerCAmelCase     :							Optional[Any]    =    num_attention_heads
														_lowerCAmelCase     :							Any    =    intermediate_size
														_lowerCAmelCase     :							List[str]    =    hidden_act
														_lowerCAmelCase     :							Union[str, Any]    =    hidden_dropout_prob
														_lowerCAmelCase     :							Any    =    attention_probs_dropout_prob
														_lowerCAmelCase     :							int    =    max_position_embeddings
														_lowerCAmelCase     :							Optional[int]    =    type_vocab_size
														_lowerCAmelCase     :							Optional[Any]    =    type_sequence_label_size
														_lowerCAmelCase     :							List[str]    =    initializer_range
														_lowerCAmelCase     :							List[Any]    =    num_labels
														_lowerCAmelCase     :							Tuple    =    scope
														_lowerCAmelCase     :							str    =    range_bbox







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    ids_tensor([self.batch_size, self.seq_length],			self.vocab_size)

														_lowerCAmelCase     :							int    =    ids_tensor([self.batch_size, self.seq_length, 4],			self.range_bbox)
														# Ensure that bbox is legal
														for i in range(bbox.shape[0]):
																					for j in range(bbox.shape[1]):
																												if bbox[i, j, 3] < bbox[i, j, 1]:
																																			_lowerCAmelCase     :							Dict    =    bbox[i, j, 3]
																																			_lowerCAmelCase     :							int    =    bbox[i, j, 1]
																																			_lowerCAmelCase     :							Tuple    =    t
																												if bbox[i, j, 2] < bbox[i, j, 0]:
																																			_lowerCAmelCase     :							str    =    bbox[i, j, 2]
																																			_lowerCAmelCase     :							List[Any]    =    bbox[i, j, 0]
																																			_lowerCAmelCase     :							str    =    t

														_lowerCAmelCase     :							Optional[Any]    =    None
														if self.use_input_mask:
																					_lowerCAmelCase     :							Union[str, Any]    =    ids_tensor([self.batch_size, self.seq_length],			vocab_size=2)

														_lowerCAmelCase     :							Dict    =    None
														if self.use_token_type_ids:
																					_lowerCAmelCase     :							Any    =    ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size)

														_lowerCAmelCase     :							Optional[int]    =    None
														_lowerCAmelCase     :							Optional[Any]    =    None
														if self.use_labels:
																					_lowerCAmelCase     :							Optional[Any]    =    ids_tensor([self.batch_size],			self.type_sequence_label_size)
																					_lowerCAmelCase     :							Any    =    ids_tensor([self.batch_size, self.seq_length],			self.num_labels)

														_lowerCAmelCase     :							Optional[int]    =    self.get_config()

														return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels







							def    snake_case__      (   self):

														'''simple docstring'''

														return LiltConfig(
														    vocab_size=self.vocab_size,			hidden_size=self.hidden_size,			num_hidden_layers=self.num_hidden_layers,			num_attention_heads=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			max_position_embeddings=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			initializer_range=self.initializer_range,			)







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    LiltModel(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Dict    =    model(__a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a)
														_lowerCAmelCase     :							str    =    model(__a,			bbox=__a,			token_type_ids=__a)
														_lowerCAmelCase     :							List[Any]    =    model(__a,			bbox=__a)

														self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size))
														self.parent.assertEqual(result.pooler_output.shape,			(self.batch_size, self.hidden_size))







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    self.num_labels
														_lowerCAmelCase     :							Optional[Any]    =    LiltForTokenClassification(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Dict    =    model(
														    __a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a,			labels=__a)
														self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.num_labels))







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a,			__a,			__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[int]    =    LiltForQuestionAnswering(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase     :							Tuple    =    model(
														    __a,			bbox=__a,			attention_mask=__a,			token_type_ids=__a,			start_positions=__a,			end_positions=__a,			)
														self.parent.assertEqual(result.start_logits.shape,			(self.batch_size, self.seq_length))
														self.parent.assertEqual(result.end_logits.shape,			(self.batch_size, self.seq_length))







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    self.prepare_config_and_inputs()
														(
														    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    (
														    _lowerCAmelCase
														)   ,    
														)     :							Dict    =    config_and_inputs
														_lowerCAmelCase     :							List[Any]    =    {
														    "input_ids": input_ids,
														    "bbox": bbox,
														    "token_type_ids": token_type_ids,
														    "attention_mask": input_mask,
														}
														return config, inputs_dict



@require_torch
class   UpperCAmelCase_    (       a   ,					a   ,					a   ,					unittest.TestCase):
							lowerCamelCase__   			=		(
							    (
							        LiltModel,
							        LiltForSequenceClassification,
							        LiltForTokenClassification,
							        LiltForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase__   			=		(
							    {
							        'feature-extraction': LiltModel,
							        'question-answering': LiltForQuestionAnswering,
							        'text-classification': LiltForSequenceClassification,
							        'token-classification': LiltForTokenClassification,
							        'zero-shot': LiltForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase__   			=		False
							lowerCamelCase__   			=		False







							def    snake_case__      (   self,			__a,			__a,			__a,			__a,			__a):

														'''simple docstring'''

														return True







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    LiltModelTester(self)
														_lowerCAmelCase     :							Union[str, Any]    =    ConfigTester(self,			config_class=__a,			hidden_size=37)







							def    snake_case__      (   self):

														'''simple docstring'''

														self.config_tester.run_common_tests()







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    self.model_tester.prepare_config_and_inputs()
														for type in ["absolute", "relative_key", "relative_key_query"]:
																					_lowerCAmelCase     :							Any    =    type
																					self.model_tester.create_and_check_model(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_token_classification(*__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[int]    =    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_question_answering(*__a)







							@slow
							def    snake_case__      (   self):

														'''simple docstring'''

														for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																					_lowerCAmelCase     :							str    =    LiltModel.from_pretrained(__a)
																					self.assertIsNotNone(__a)



@require_torch
@slow
class   UpperCAmelCase_    (       unittest.TestCase):







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base").to(__a)

														_lowerCAmelCase     :							Any    =    torch.tensor([[1, 2]],			device=__a)
														_lowerCAmelCase     :							str    =    torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]],			device=__a)

														# forward pass
														with torch.no_grad():
																					_lowerCAmelCase     :							Optional[Any]    =    model(input_ids=__a,			bbox=__a)

														_lowerCAmelCase     :							Optional[int]    =    torch.Size([1, 2, 768])
														_lowerCAmelCase     :							List[str]    =    torch.tensor(
														    [[-0.0_653, 0.0_950, -0.0_061], [-0.0_545, 0.0_926, -0.0_324]],			device=__a,			)

														self.assertTrue(outputs.last_hidden_state.shape,			__a)
														self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3],			__a,			atol=1E-3))
