






import math
from collections import OrderedDict

import torch
from packaging import version
from torch import Tensor, nn

from .utils import logging


__UpperCAmelCase      =				logging.get_logger(__name__)
class 	a_(		nn.Module	):



     """simple docstring"""



     def __init__(		self					:	Optional[Any])							->	Optional[Any]:

      """simple docstring"""
      super().__init__()
      if version.parse(torch.__version__) < version.parse('1.12.0'):
       raise ImportError(
           f'''You are using torch=={torch.__version__}, but torch>=1.12.0 is required to use '''
           'PytorchGELUTanh. Please upgrade torch.')





     def     __UpperCamelCase						(		self					:	Optional[Any]  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return nn.functional.gelu(_A  ,  approximate='tanh')

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Optional[Any]  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.04_47_15 * torch.pow(_A  ,  3.0))))

class 	a_(		nn.Module	):



     """simple docstring"""



     def __init__(		self					:	Optional[int]  ,  lowerCAmelCase__					:	bool = False)							->	Tuple:

      """simple docstring"""
      super().__init__()
      if use_gelu_python:
       SCREAMING_SNAKE_CASE				    =		self._gelu_python
      else:
       SCREAMING_SNAKE_CASE				    =		nn.functional.gelu





     def     __UpperCamelCase						(		self					:	int  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))





     def     __UpperCamelCase						(		self					:	Optional[int]  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return self.act(_A)

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Tuple  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return 0.5 * input * (1.0 + torch.tanh(input * 0.79_78_84_56_08 * (1.0 + 0.04_47_15 * input * input)))

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Optional[int]  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return input * torch.sigmoid(1.7_02 * input)

class 	a_(		nn.Module	):



     """simple docstring"""



     def __init__(		self					:	List[str]  ,  lowerCAmelCase__					:	float  ,  lowerCAmelCase__					:	float)							->	List[Any]:

      """simple docstring"""



      if min > max:
       raise ValueError(f'''min should be < max (got min: {min}, max: {max})''')

      super().__init__()
      SCREAMING_SNAKE_CASE				    =		min
      SCREAMING_SNAKE_CASE				    =		max





     def     __UpperCamelCase						(		self					:	Dict  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return torch.clip(gelu(_A)  ,  self.min  ,  self.max)

class 	a_(		nn.Module	):



     """simple docstring"""



     def __init__(		self					:	Dict)							->	Optional[int]:

      """simple docstring"""
      super().__init__()
      SCREAMING_SNAKE_CASE				    =		math.sqrt(2 / math.pi)





     def     __UpperCamelCase						(		self					:	Tuple  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.04_47_15 * torch.pow(_A  ,  3))))

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	int  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return nn.functional.silu(_A)

class 	a_(		nn.Module	):



     """simple docstring"""



     def __init__(		self					:	Any)							->	Optional[int]:

      """simple docstring"""
      super().__init__()
      if version.parse(torch.__version__) < version.parse('1.9.0'):
       SCREAMING_SNAKE_CASE				    =		self._mish_python
      else:
       SCREAMING_SNAKE_CASE				    =		nn.functional.mish





     def     __UpperCamelCase						(		self					:	Optional[Any]  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return input * torch.tanh(nn.functional.softplus(_A))





     def     __UpperCamelCase						(		self					:	Dict  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return self.act(_A)

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	str  ,  lowerCAmelCase__					:	Tensor)							->	Tensor:

      """simple docstring"""
      return input

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	List[str]  ,  lowerCAmelCase__					:	Optional[Any]  ,  lowerCAmelCase__					:	Tuple=0.70_71_07  ,  lowerCAmelCase__					:	List[str]=0.28_20_95)							->	Union[str, Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		(input - mu).div(sigma * math.sqrt(2.0))
      return 0.5 * (1.0 + torch.erf(_A))

class 	a_(		nn.Module	):



     """simple docstring"""



     def     __UpperCamelCase						(		self					:	Tuple  ,  lowerCAmelCase__					:	Any)							->	Optional[Any]:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		nn.functional.relu(_A)
      SCREAMING_SNAKE_CASE				    =		torch.square(_A)
      return squared

class 	a_(		lowercase__	):



     """simple docstring"""



     def __getitem__(		self					:	Any  ,  lowerCAmelCase__					:	Optional[int])							->	Dict:

      """simple docstring"""
      SCREAMING_SNAKE_CASE				    =		super().__getitem__(_A)
      SCREAMING_SNAKE_CASE    ,   SCREAMING_SNAKE_CASE				    =		content if isinstance(_A  ,  _A) else (content, {})
      return cls(**_A)

__UpperCAmelCase      =				{
    "gelu": GELUActivation,
    "gelu_10": (ClippedGELUActivation, {"min": -1_0, "max": 1_0}),
    "gelu_fast": FastGELUActivation,
    "gelu_new": NewGELUActivation,
    "gelu_python": (GELUActivation, {"use_gelu_python": True}),
    "gelu_pytorch_tanh": PytorchGELUTanh,
    "gelu_accurate": AccurateGELUActivation,
    "laplace": LaplaceActivation,
    "linear": LinearActivation,
    "mish": MishActivation,
    "quick_gelu": QuickGELUActivation,
    "relu": nn.ReLU,
    "relu2": ReLUSquaredActivation,
    "relu6": nn.ReLUa,
    "sigmoid": nn.Sigmoid,
    "silu": SiLUActivation,
    "swish": SiLUActivation,
    "tanh": nn.Tanh,
}
__UpperCAmelCase      =				ClassInstantier(ACTaCLS)





def 						A_					(						lowercase_			)				->Any:







 """simple docstring"""

 if activation_string in ACTaFN:
  return ACTaFN[activation_string]
 else:
  raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()			)}'''			)


# For backwards compatibility with: from activations import gelu_python
__UpperCAmelCase      =				get_activation("gelu_python")
__UpperCAmelCase      =				get_activation("gelu_new")
__UpperCAmelCase      =				get_activation("gelu")
__UpperCAmelCase      =				get_activation("gelu_fast")
__UpperCAmelCase      =				get_activation("quick_gelu")
__UpperCAmelCase      =				get_activation("silu")
__UpperCAmelCase      =				get_activation("mish")
__UpperCAmelCase      =				get_activation("linear")






'''simple docstring'''





import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MarkupLMForQuestionAnswering,
     MarkupLMForSequenceClassification,
     MarkupLMForTokenClassification,
     MarkupLMModel,
 )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer


class 			UpperCamelCase__	:






 def __init__(      self	:str						,					_A	:Any						,					_A	:Union[str, Any]=13						,					_A	:List[Any]=7						,					_A	:int=True						,					_A	:Union[str, Any]=True						,					_A	:List[Any]=True						,					_A	:Dict=True						,					_A	:int=99						,					_A	:Union[str, Any]=32						,					_A	:int=5						,					_A	:List[str]=4						,					_A	:Optional[int]=37						,					_A	:List[str]="gelu"						,					_A	:Any=0.1						,					_A	:int=0.1						,					_A	:List[str]=512						,					_A	:Optional[Any]=16						,					_A	:Optional[Any]=2						,					_A	:List[str]=0.02						,					_A	:Any=3						,					_A	:Tuple=None						,					_A	:str=20						,					_A	:List[Any]=30						,					_A	:Dict=2						,					_A	:Union[str, Any]=2						,					_A	:int=10						,					)					->      Any:







       '''simple docstring'''



       __A													=      parent
       __A													=      batch_size
       __A													=      seq_length
       __A													=      is_training
       __A													=      use_input_mask
       __A													=      use_token_type_ids
       __A													=      use_labels
       __A													=      vocab_size
       __A													=      hidden_size
       __A													=      num_hidden_layers
       __A													=      num_attention_heads
       __A													=      intermediate_size
       __A													=      hidden_act
       __A													=      hidden_dropout_prob
       __A													=      attention_probs_dropout_prob
       __A													=      max_position_embeddings
       __A													=      type_vocab_size
       __A													=      type_sequence_label_size
       __A													=      initializer_range
       __A													=      num_labels
       __A													=      scope
       __A													=      max_xpath_tag_unit_embeddings
       __A													=      max_xpath_subs_unit_embeddings
       __A													=      tag_pad_id
       __A													=      subs_pad_id
       __A													=      max_depth






 def 						lowercase_    (      self	:Tuple      )					->      Optional[int]:







       '''simple docstring'''



       __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.vocab_size      )

       __A													=      ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]						,					self.max_xpath_tag_unit_embeddings      )

       __A													=      ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]						,					self.max_xpath_subs_unit_embeddings      )

       __A													=      None
       if self.use_input_mask:
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					vocab_size=2      )

       __A													=      None
       if self.use_token_type_ids:
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.type_vocab_size      )

       __A													=      None
       __A													=      None
       if self.use_labels:
             __A													=      ids_tensor([self.batch_size]						,					self.type_sequence_label_size      )
             __A													=      ids_tensor([self.batch_size, self.seq_length]						,					self.num_labels      )

       __A													=      self.get_config()

       return (
           config,
           input_ids,
           xpath_tags_seq,
           xpath_subs_seq,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
       )






 def 						lowercase_    (      self	:List[Any]      )					->      int:







       '''simple docstring'''



       return MarkupLMConfig(
           vocab_size=self.vocab_size						,					hidden_size=self.hidden_size						,					num_hidden_layers=self.num_hidden_layers						,					num_attention_heads=self.num_attention_heads						,					intermediate_size=self.intermediate_size						,					hidden_act=self.hidden_act						,					hidden_dropout_prob=self.hidden_dropout_prob						,					attention_probs_dropout_prob=self.attention_probs_dropout_prob						,					max_position_embeddings=self.max_position_embeddings						,					type_vocab_size=self.type_vocab_size						,					initializer_range=self.initializer_range						,					max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings						,					max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings						,					tag_pad_id=self.tag_pad_id						,					subs_pad_id=self.subs_pad_id						,					max_depth=self.max_depth						,					)






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:List[str]						,					_A	:str						,					_A	:int						,					_A	:str						,					_A	:Dict						,					_A	:str						,					_A	:Optional[int]						,					_A	:Optional[Any]						,					)					->      Tuple:







       '''simple docstring'''



       __A													=      MarkupLMModel(config=_A      )
       model.to(_A      )
       model.eval()
       print('Configs:'						,					model.config.tag_pad_id						,					model.config.subs_pad_id      )
       __A													=      model(_A						,					attention_mask=_A						,					token_type_ids=_A      )
       __A													=      model(_A						,					token_type_ids=_A      )
       __A													=      model(_A      )
       self.parent.assertEqual(result.last_hidden_state.shape						,					(self.batch_size, self.seq_length, self.hidden_size)      )
       self.parent.assertEqual(result.pooler_output.shape						,					(self.batch_size, self.hidden_size)      )






 def 						lowercase_    (      self	:Any						,					_A	:Optional[Any]						,					_A	:Any						,					_A	:Optional[int]						,					_A	:Any						,					_A	:int						,					_A	:List[Any]						,					_A	:Dict						,					_A	:List[str]						,					)					->      Any:







       '''simple docstring'''



       __A													=      self.num_labels
       __A													=      MarkupLMForSequenceClassification(_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					labels=_A						,					)
       self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.num_labels)      )






 def 						lowercase_    (      self	:Any						,					_A	:List[str]						,					_A	:str						,					_A	:str						,					_A	:List[Any]						,					_A	:Optional[Any]						,					_A	:Union[str, Any]						,					_A	:Dict						,					_A	:Union[str, Any]						,					)					->      Dict:







       '''simple docstring'''



       __A													=      self.num_labels
       __A													=      MarkupLMForTokenClassification(config=_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					labels=_A						,					)
       self.parent.assertEqual(result.logits.shape						,					(self.batch_size, self.seq_length, self.num_labels)      )






 def 						lowercase_    (      self	:int						,					_A	:str						,					_A	:Tuple						,					_A	:Any						,					_A	:int						,					_A	:Any						,					_A	:Any						,					_A	:Union[str, Any]						,					_A	:List[Any]						,					)					->      List[str]:







       '''simple docstring'''



       __A													=      MarkupLMForQuestionAnswering(config=_A      )
       model.to(_A      )
       model.eval()
       __A													=      model(
           _A						,					xpath_tags_seq=_A						,					xpath_subs_seq=_A						,					attention_mask=_A						,					token_type_ids=_A						,					start_positions=_A						,					end_positions=_A						,					)
       self.parent.assertEqual(result.start_logits.shape						,					(self.batch_size, self.seq_length)      )
       self.parent.assertEqual(result.end_logits.shape						,					(self.batch_size, self.seq_length)      )






 def 						lowercase_    (      self	:Dict      )					->      List[str]:







       '''simple docstring'''



       __A													=      self.prepare_config_and_inputs()
       (
           (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   (
           __A
       )						,   
       )													=      config_and_inputs
       __A													=      {
           'input_ids': input_ids,
           'xpath_tags_seq': xpath_tags_seq,
           'xpath_subs_seq': xpath_subs_seq,
           'token_type_ids': token_type_ids,
           'attention_mask': input_mask,
       }
       return config, inputs_dict


@require_torch
class 			UpperCamelCase__	(     SCREAMING_SNAKE_CASE   ,  SCREAMING_SNAKE_CASE   ,  unittest.TestCase):
 UpperCAmelCase__     :     List[Any]    					=       (
     (
         MarkupLMModel,
         MarkupLMForSequenceClassification,
         MarkupLMForTokenClassification,
         MarkupLMForQuestionAnswering,
     )
     if is_torch_available()
     else None
 )
 UpperCAmelCase__     :     Tuple    					=       (
     {
         'feature-extraction': MarkupLMModel,
         'question-answering': MarkupLMForQuestionAnswering,
         'text-classification': MarkupLMForSequenceClassification,
         'token-classification': MarkupLMForTokenClassification,
         'zero-shot': MarkupLMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:List[str]						,					_A	:Optional[Any]						,					_A	:Dict						,					_A	:Optional[Any]						,					_A	:Optional[int]      )					->      Tuple:







       '''simple docstring'''



       return True






 def 						lowercase_    (      self	:List[str]      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      MarkupLMModelTester(self      )
       __A													=      ConfigTester(self						,					config_class=_A						,					hidden_size=37      )






 def 						lowercase_    (      self	:Optional[Any]      )					->      int:







       '''simple docstring'''



       self.config_tester.run_common_tests()






 def 						lowercase_    (      self	:Union[str, Any]      )					->      Optional[Any]:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_A      )






 def 						lowercase_    (      self	:Optional[Any]      )					->      Any:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_A      )






 def 						lowercase_    (      self	:Tuple      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_A      )






 def 						lowercase_    (      self	:List[Any]      )					->      Tuple:







       '''simple docstring'''



       __A													=      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_A      )





def 							snake_case				(		)->      str:







      """simple docstring"""
      __A													=      '\n    <!DOCTYPE html>\n    <html>\n    <head>\n    <title>Page Title</title>\n    </head>\n    <body>\n\n    <h1>This is a Heading</h1>\n    <p>This is a paragraph.</p>\n\n    </body>\n    </html>\n    '

      return html_string


@require_torch
class 			UpperCamelCase__	(     unittest.TestCase):






 @cached_property
 def 						lowercase_    (      self	:Optional[int]      )					->      Dict:







       '''simple docstring'''



       __A													=      MarkupLMFeatureExtractor()
       __A													=      MarkupLMTokenizer.from_pretrained('microsoft/markuplm-base'      )

       return MarkupLMProcessor(_A						,					_A      )






 @slow
 def 						lowercase_    (      self	:Any      )					->      str:







       '''simple docstring'''



       __A													=      MarkupLMModel.from_pretrained('microsoft/markuplm-base'      ).to(_A      )

       __A													=      self.default_processor

       __A													=      processor(prepare_html_string()						,					return_tensors='pt'      )
       __A													=      inputs.to(_A      )

       # forward pass
       with torch.no_grad():
             __A													=      model(**_A      )

       # verify the last hidden states
       __A													=      torch.Size([1, 14, 768]      )
       self.assertEqual(outputs.last_hidden_state.shape						,					_A      )

       __A													=      torch.tensor(
           [[0.0_675, -0.0_052, 0.5_001], [-0.2_281, 0.0_802, 0.2_192], [-0.0_583, -0.3_311, 0.1_185]]      ).to(_A      )

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]						,					_A						,					atol=1E-4      )      )






