

"""simple docstring"""





import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a					 =				logging.get_logger(__name__)

_a					 =				{
    'source_spm': 'source.spm',
    'target_spm': 'target.spm',
    'vocab': 'vocab.json',
    'target_vocab_file': 'target_vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
}

_a					 =				{
    'source_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm'
    },
    'target_spm': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm'
    },
    'vocab': {
        'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json'
    },
    'tokenizer_config_file': {
        'Helsinki-NLP/opus-mt-en-de': (
            'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json'
        )
    },
}

_a					 =				{'Helsinki-NLP/opus-mt-en-de': 5_12}
_a					 =				{}




class    _lowerCAmelCase    (      lowercase  ):







 """simple docstring"""







 __UpperCAmelCase      :   Dict  						=      VOCAB_FILES_NAMES
 __UpperCAmelCase      :   Union[str, Any]  						=      PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase      :   Any  						=      PRETRAINED_INIT_CONFIGURATION
 __UpperCAmelCase      :   List[Any]  						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase      :   Any  						=      ["input_ids", "attention_mask"]
 __UpperCAmelCase      :   Any  						=      re.compile(">>.+<<"  )  # type: re.Pattern


 def __init__(     self				:     Dict,   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     int,   UpperCAmelCase__				:     List[str]=None,   UpperCAmelCase__				:     Union[str, Any]=None,   UpperCAmelCase__				:     Optional[Any]=None,   UpperCAmelCase__				:     int="<unk>",   UpperCAmelCase__				:     Tuple="</s>",   UpperCAmelCase__				:     Optional[Any]="<pad>",   UpperCAmelCase__				:     Tuple=5_1_2,   UpperCAmelCase__				:     Optional[Dict[str, Any]] = None,   UpperCAmelCase__				:     Optional[Any]=False,   **UpperCAmelCase__				:     Optional[int],   ):
    __lowercase								=		{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
        source_lang=UpperCAmelCase__,   target_lang=UpperCAmelCase__,   unk_token=UpperCAmelCase__,   eos_token=UpperCAmelCase__,   pad_token=UpperCAmelCase__,   model_max_length=UpperCAmelCase__,   sp_model_kwargs=self.sp_model_kwargs,   target_vocab_file=UpperCAmelCase__,   separate_vocabs=UpperCAmelCase__,   **UpperCAmelCase__,   )
    assert Path(UpperCAmelCase__      ).exists(), F"""cannot find spm source {source_spm}"""

    __lowercase								=		separate_vocabs
    __lowercase								=		load_json(UpperCAmelCase__      )
    if self.unk_token not in self.encoder:
       raise KeyError("<unk> token must be in vocab"      )
    assert self.pad_token in self.encoder

    if separate_vocabs:
       __lowercase								=		load_json(UpperCAmelCase__      )
       __lowercase								=		{v: k for k, v in self.target_encoder.items()}
       __lowercase								=		[]
    else:
       __lowercase								=		{v: k for k, v in self.encoder.items()}
       __lowercase								=		[k for k in self.encoder if k.startswith(">>"      ) and k.endswith("<<"      )]

    __lowercase								=		source_lang
    __lowercase								=		target_lang
    __lowercase								=		[source_spm, target_spm]

    # load SentencePiece model for pre-processing
    __lowercase								=		load_spm(UpperCAmelCase__,   self.sp_model_kwargs      )
    __lowercase								=		load_spm(UpperCAmelCase__,   self.sp_model_kwargs      )
    __lowercase								=		self.spm_source
    __lowercase								=		self.encoder

    # Multilingual target side: default to using first supported language code.

    self._setup_normalizer()


 def    _lowercase					(     self				:     str      ):
    try:
       from sacremoses import MosesPunctNormalizer

       __lowercase								=		MosesPunctNormalizer(self.source_lang      ).normalize
    except (ImportError, FileNotFoundError):
       warnings.warn("Recommended: pip install sacremoses."      )
       __lowercase								=		lambda UpperCAmelCase__      : x


 def    _lowercase					(     self				:     Optional[int],   UpperCAmelCase__				:     str      ):
    return self.punc_normalizer(UpperCAmelCase__      ) if x else ""


 def    _lowercase					(     self				:     Dict,   UpperCAmelCase__				:     Tuple      ):
    return self.current_encoder.get(UpperCAmelCase__,   self.current_encoder[self.unk_token]      )


 def    _lowercase					(     self				:     int,   UpperCAmelCase__				:     str      ):
    __lowercase								=		self.language_code_re.match(UpperCAmelCase__      )
    __lowercase								=		[match.group(0      )] if match else []
    return code, self.language_code_re.sub("",   UpperCAmelCase__      )


 def    _lowercase					(     self				:     int,   UpperCAmelCase__				:     str      ):
    __lowercase						,__lowercase								=		self.remove_language_code(UpperCAmelCase__      )
    __lowercase								=		self.current_spm.encode(UpperCAmelCase__,   out_type=UpperCAmelCase__      )
    return code + pieces


 def    _lowercase					(     self				:     str,   UpperCAmelCase__				:     int      ):
    return self.decoder.get(UpperCAmelCase__,   self.unk_token      )


 def    _lowercase					(     self				:     Any,   UpperCAmelCase__				:     Any,   **UpperCAmelCase__				:     List[Any]      ):
    return super().batch_decode(UpperCAmelCase__,   **UpperCAmelCase__      )


 def    _lowercase					(     self				:     Union[str, Any],   UpperCAmelCase__				:     int,   **UpperCAmelCase__				:     List[Any]      ):
    return super().decode(UpperCAmelCase__,   **UpperCAmelCase__      )


 def    _lowercase					(     self				:     Any,   UpperCAmelCase__				:     List[str]      ):
    __lowercase								=		self.spm_source if self._decode_use_source_tokenizer else self.spm_target
    __lowercase								=		[]
    __lowercase								=		""
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          out_string += sp_model.decode_pieces(UpperCAmelCase__      ) + token + " "
          __lowercase								=		[]
       else:
          current_sub_tokens.append(UpperCAmelCase__      )
    out_string += sp_model.decode_pieces(UpperCAmelCase__      )
    return out_string.strip()


 def    _lowercase					(     self				:     Tuple,   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     Optional[Any]=None      ):
    if token_ids_a is None:
       return token_ids_a + [self.eos_token_id]
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return token_ids_a + token_ids_a + [self.eos_token_id]


 def    _lowercase					(     self				:     Union[str, Any]      ):
    __lowercase								=		self.spm_source
    __lowercase								=		self.encoder


 def    _lowercase					(     self				:     List[str]      ):
    __lowercase								=		self.spm_target
    if self.separate_vocabs:
       __lowercase								=		self.target_encoder


 @property
 def    _lowercase					(     self				:     List[str]      ):
    return len(self.encoder      )


 def    _lowercase					(     self				:     Dict,   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     Optional[str] = None      ):
    if not os.path.isdir(UpperCAmelCase__      ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
       return
    __lowercase								=		[]

    if self.separate_vocabs:
       __lowercase								=		os.path.join(
           UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"],   )
       __lowercase								=		os.path.join(
           UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["target_vocab_file"],   )
       save_json(self.encoder,   UpperCAmelCase__      )
       save_json(self.target_encoder,   UpperCAmelCase__      )
       saved_files.append(UpperCAmelCase__      )
       saved_files.append(UpperCAmelCase__      )
    else:
       __lowercase								=		os.path.join(
           UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"]      )
       save_json(self.encoder,   UpperCAmelCase__      )
       saved_files.append(UpperCAmelCase__      )

    for spm_save_filename, spm_orig_path, spm_model in zip(
        [VOCAB_FILES_NAMES["source_spm"], VOCAB_FILES_NAMES["target_spm"]],   self.spm_files,   [self.spm_source, self.spm_target],   ):
       __lowercase								=		os.path.join(
           UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + spm_save_filename      )
       if os.path.abspath(UpperCAmelCase__      ) != os.path.abspath(UpperCAmelCase__      ) and os.path.isfile(UpperCAmelCase__      ):
          copyfile(UpperCAmelCase__,   UpperCAmelCase__      )
          saved_files.append(UpperCAmelCase__      )
       elif not os.path.isfile(UpperCAmelCase__      ):
          with open(UpperCAmelCase__,   "wb"      ) as fi:
             __lowercase								=		spm_model.serialized_model_proto()
             fi.write(UpperCAmelCase__      )
          saved_files.append(UpperCAmelCase__      )

    return tuple(UpperCAmelCase__      )


 def    _lowercase					(     self				:     List[Any]      ):
    return self.get_src_vocab()


 def    _lowercase					(     self				:     List[str]      ):
    return dict(self.encoder,   **self.added_tokens_encoder      )


 def    _lowercase					(     self				:     Any      ):
    return dict(self.target_encoder,   **self.added_tokens_decoder      )


 def __getstate__(     self				:     int      ):
    __lowercase								=		self.__dict__.copy()
    state.update(
        {k: None for k in ["spm_source", "spm_target", "current_spm", "punc_normalizer", "target_vocab_file"]}      )
    return state


 def __setstate__(     self				:     Any,   UpperCAmelCase__				:     Dict      ):
    __lowercase								=		d

    # for backward compatibility
    if not hasattr(self,   "sp_model_kwargs"      ):
       __lowercase								=		{}

    __lowercase						,__lowercase								=		(load_spm(UpperCAmelCase__,   self.sp_model_kwargs      ) for f in self.spm_files)
    __lowercase								=		self.spm_source
    self._setup_normalizer()


 def    _lowercase					(     self				:     Optional[Any],   *UpperCAmelCase__				:     List[str],   **UpperCAmelCase__				:     List[Any]      ):
    return 1


 def    _lowercase					(     self				:     Optional[int],   UpperCAmelCase__				:     Any      ):
    __lowercase								=		set(self.all_special_ids      )  # call it once instead of inside list comp
    all_special_ids.remove(self.unk_token_id      )  # <unk> is only sometimes special
    return [1 if x in all_special_ids else 0 for x in seq]



 def    _lowercase					(     self				:     List[str],   UpperCAmelCase__				:     List,   UpperCAmelCase__				:     Optional[List] = None,   UpperCAmelCase__				:     bool = False      ):
    if already_has_special_tokens:
       return self._special_token_mask(UpperCAmelCase__      )
    elif token_ids_a is None:
       return self._special_token_mask(UpperCAmelCase__      ) + [1]
    else:
       return self._special_token_mask(token_ids_a + token_ids_a      ) + [1]







def     _A    (    UpperCamelCase_	:			str,     UpperCamelCase_	:			Dict[str, Any])      ->		sentencepiece.SentencePieceProcessor:
   '''simple docstring'''


   __lowercase								=		sentencepiece.SentencePieceProcessor(**UpperCamelCase_)
   spm.Load(UpperCamelCase_)
   return spm




def     _A    (    UpperCamelCase_	:			List[str],     UpperCamelCase_	:			str)      ->		None:
   '''simple docstring'''


   with open(UpperCamelCase_,     "w") as f:
      json.dump(UpperCamelCase_,     UpperCamelCase_,     indent=2)




def     _A    (    UpperCamelCase_	:			str)      ->		Union[Dict, List]:
   '''simple docstring'''


   with open(UpperCamelCase_,     "r") as f:
      return json.load(UpperCamelCase_)







"""simple docstring"""





import unittest

from transformers import LayoutLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       LayoutLMForMaskedLM,
       LayoutLMForQuestionAnswering,
       LayoutLMForSequenceClassification,
       LayoutLMForTokenClassification,
       LayoutLMModel,
   )




class    _lowerCAmelCase    :







 """simple docstring"""







 def __init__(     self				:     Optional[int],   UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     Dict=1_3,   UpperCAmelCase__				:     Tuple=7,   UpperCAmelCase__				:     Union[str, Any]=True,   UpperCAmelCase__				:     Tuple=True,   UpperCAmelCase__				:     Dict=True,   UpperCAmelCase__				:     List[str]=True,   UpperCAmelCase__				:     Optional[int]=9_9,   UpperCAmelCase__				:     Union[str, Any]=3_2,   UpperCAmelCase__				:     str=5,   UpperCAmelCase__				:     int=4,   UpperCAmelCase__				:     Any=3_7,   UpperCAmelCase__				:     Tuple="gelu",   UpperCAmelCase__				:     str=0.1,   UpperCAmelCase__				:     Tuple=0.1,   UpperCAmelCase__				:     Any=5_1_2,   UpperCAmelCase__				:     Optional[Any]=1_6,   UpperCAmelCase__				:     Optional[Any]=2,   UpperCAmelCase__				:     Tuple=0.02,   UpperCAmelCase__				:     List[str]=3,   UpperCAmelCase__				:     Any=4,   UpperCAmelCase__				:     Dict=None,   UpperCAmelCase__				:     Dict=1_0_0_0,   ):
    __lowercase								=		parent
    __lowercase								=		batch_size
    __lowercase								=		seq_length
    __lowercase								=		is_training
    __lowercase								=		use_input_mask
    __lowercase								=		use_token_type_ids
    __lowercase								=		use_labels
    __lowercase								=		vocab_size
    __lowercase								=		hidden_size
    __lowercase								=		num_hidden_layers
    __lowercase								=		num_attention_heads
    __lowercase								=		intermediate_size
    __lowercase								=		hidden_act
    __lowercase								=		hidden_dropout_prob
    __lowercase								=		attention_probs_dropout_prob
    __lowercase								=		max_position_embeddings
    __lowercase								=		type_vocab_size
    __lowercase								=		type_sequence_label_size
    __lowercase								=		initializer_range
    __lowercase								=		num_labels
    __lowercase								=		num_choices
    __lowercase								=		scope
    __lowercase								=		range_bbox


 def    _lowercase					(     self				:     Union[str, Any]      ):
    __lowercase								=		ids_tensor([self.batch_size, self.seq_length],   self.vocab_size      )

    __lowercase								=		ids_tensor([self.batch_size, self.seq_length, 4],   self.range_bbox      )
    # Ensure that bbox is legal
    for i in range(bbox.shape[0]      ):
       for j in range(bbox.shape[1]      ):
          if bbox[i, j, 3] < bbox[i, j, 1]:
             __lowercase								=		bbox[i, j, 3]
             __lowercase								=		bbox[i, j, 1]
             __lowercase								=		t
          if bbox[i, j, 2] < bbox[i, j, 0]:
             __lowercase								=		bbox[i, j, 2]
             __lowercase								=		bbox[i, j, 0]
             __lowercase								=		t

    __lowercase								=		None
    if self.use_input_mask:
       __lowercase								=		ids_tensor([self.batch_size, self.seq_length],   vocab_size=2      )

    __lowercase								=		None
    if self.use_token_type_ids:
       __lowercase								=		ids_tensor([self.batch_size, self.seq_length],   self.type_vocab_size      )

    __lowercase								=		None
    __lowercase								=		None
    __lowercase								=		None
    if self.use_labels:
       __lowercase								=		ids_tensor([self.batch_size],   self.type_sequence_label_size      )
       __lowercase								=		ids_tensor([self.batch_size, self.seq_length],   self.num_labels      )
       __lowercase								=		ids_tensor([self.batch_size],   self.num_choices      )

    __lowercase								=		self.get_config()

    return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


 def    _lowercase					(     self				:     Tuple      ):
    return LayoutLMConfig(
        vocab_size=self.vocab_size,   hidden_size=self.hidden_size,   num_hidden_layers=self.num_hidden_layers,   num_attention_heads=self.num_attention_heads,   intermediate_size=self.intermediate_size,   hidden_act=self.hidden_act,   hidden_dropout_prob=self.hidden_dropout_prob,   attention_probs_dropout_prob=self.attention_probs_dropout_prob,   max_position_embeddings=self.max_position_embeddings,   type_vocab_size=self.type_vocab_size,   initializer_range=self.initializer_range,   )


 def    _lowercase					(     self				:     Optional[Any],   UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     List[Any],   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     List[str],   UpperCAmelCase__				:     Any      ):
    __lowercase								=		LayoutLMModel(config=UpperCAmelCase__      )
    model.to(UpperCAmelCase__      )
    model.eval()
    __lowercase								=		model(UpperCAmelCase__,   UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__      )
    __lowercase								=		model(UpperCAmelCase__,   UpperCAmelCase__,   token_type_ids=UpperCAmelCase__      )
    __lowercase								=		model(UpperCAmelCase__,   UpperCAmelCase__      )
    self.parent.assertEqual(result.last_hidden_state.shape,   (self.batch_size, self.seq_length, self.hidden_size)      )
    self.parent.assertEqual(result.pooler_output.shape,   (self.batch_size, self.hidden_size)      )


 def    _lowercase					(     self				:     Tuple,   UpperCAmelCase__				:     Tuple,   UpperCAmelCase__				:     List[Any],   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     Dict      ):
    __lowercase								=		LayoutLMForMaskedLM(config=UpperCAmelCase__      )
    model.to(UpperCAmelCase__      )
    model.eval()
    __lowercase								=		model(UpperCAmelCase__,   UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   labels=UpperCAmelCase__      )
    self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.vocab_size)      )


 def    _lowercase					(     self				:     Tuple,   UpperCAmelCase__				:     int,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     List[Any],   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     List[str],   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     Optional[int]      ):
    __lowercase								=		self.num_labels
    __lowercase								=		LayoutLMForSequenceClassification(UpperCAmelCase__      )
    model.to(UpperCAmelCase__      )
    model.eval()
    __lowercase								=		model(
        UpperCAmelCase__,   UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   labels=UpperCAmelCase__      )
    self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.num_labels)      )


 def    _lowercase					(     self				:     Any,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     List[Any],   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Any      ):
    __lowercase								=		self.num_labels
    __lowercase								=		LayoutLMForTokenClassification(config=UpperCAmelCase__      )
    model.to(UpperCAmelCase__      )
    model.eval()
    __lowercase								=		model(UpperCAmelCase__,   UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   labels=UpperCAmelCase__      )
    self.parent.assertEqual(result.logits.shape,   (self.batch_size, self.seq_length, self.num_labels)      )


 def    _lowercase					(     self				:     Optional[Any],   UpperCAmelCase__				:     int,   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     List[Any],   UpperCAmelCase__				:     List[str],   UpperCAmelCase__				:     int,   UpperCAmelCase__				:     Dict      ):
    __lowercase								=		LayoutLMForQuestionAnswering(config=UpperCAmelCase__      )
    model.to(UpperCAmelCase__      )
    model.eval()
    __lowercase								=		model(
        UpperCAmelCase__,   bbox=UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   start_positions=UpperCAmelCase__,   end_positions=UpperCAmelCase__,   )
    self.parent.assertEqual(result.start_logits.shape,   (self.batch_size, self.seq_length)      )
    self.parent.assertEqual(result.end_logits.shape,   (self.batch_size, self.seq_length)      )



 def    _lowercase					(     self				:     Optional[int]      ):
    __lowercase								=		self.prepare_config_and_inputs()
    (
        (
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,(
        __lowercase
    )						,
    )								=		config_and_inputs
    __lowercase								=		{
        "input_ids": input_ids,
        "bbox": bbox,
        "token_type_ids": token_type_ids,
        "attention_mask": input_mask,
    }
    return config, inputs_dict







@require_torch
class    _lowerCAmelCase    (      lowercase	,lowercase	,unittest.TestCase  ):







 """simple docstring"""







 __UpperCAmelCase      :   Dict  						=      (
     (
         LayoutLMModel,
         LayoutLMForMaskedLM,
         LayoutLMForSequenceClassification,
         LayoutLMForTokenClassification,
         LayoutLMForQuestionAnswering,
     )
     if is_torch_available()
     else None
 )
 __UpperCAmelCase      :   Union[str, Any]  						=      (
     {
         "document-question-answering": LayoutLMForQuestionAnswering,
         "feature-extraction": LayoutLMModel,
         "fill-mask": LayoutLMForMaskedLM,
         "text-classification": LayoutLMForSequenceClassification,
         "token-classification": LayoutLMForTokenClassification,
         "zero-shot": LayoutLMForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __UpperCAmelCase      :   int  						=      True


 def    _lowercase					(     self				:     str      ):
    __lowercase								=		LayoutLMModelTester(self      )
    __lowercase								=		ConfigTester(self,   config_class=UpperCAmelCase__,   hidden_size=3_7      )


 def    _lowercase					(     self				:     int      ):
    self.config_tester.run_common_tests()


 def    _lowercase					(     self				:     List[str]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*UpperCAmelCase__      )


 def    _lowercase					(     self				:     Optional[int]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
       __lowercase								=		type
       self.model_tester.create_and_check_model(*UpperCAmelCase__      )


 def    _lowercase					(     self				:     List[Any]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__      )


 def    _lowercase					(     self				:     Optional[Any]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__      )


 def    _lowercase					(     self				:     Union[str, Any]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase__      )


 def    _lowercase					(     self				:     Optional[int]      ):
    __lowercase								=		self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__      )



 @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."      )
 def    _lowercase					(     self				:     Tuple      ):
    pass







def     _A    (    )      ->		Optional[int]:
   '''simple docstring'''


   __lowercase								=		torch.tensor([[101,1019,1014,1016,1037,12849,4747,1004,14246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,11300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,19274,2772,6205,27814,16147,16147,4343,2047,10283,10969,14389,1012,2338,102]],     device=UpperCamelCase_)  # noqa: E231
   __lowercase								=		torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],],     device=UpperCamelCase_)  # noqa: E231
   __lowercase								=		torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]],     device=UpperCamelCase_)  # noqa: E231
   __lowercase								=		torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]],     device=UpperCamelCase_)  # noqa: E231
   # these are sequence labels (i.e. at the token level)
   __lowercase								=		torch.tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]],     device=UpperCamelCase_)  # noqa: E231
   # fmt: on

   return input_ids, attention_mask, bbox, token_type_ids, labels




@require_torch
class    _lowerCAmelCase    (      unittest.TestCase  ):







 """simple docstring"""







 @slow
 def    _lowercase					(     self				:     Dict      ):
    __lowercase								=		LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased"      ).to(UpperCAmelCase__      )

    __lowercase						,__lowercase						,__lowercase						,__lowercase						,__lowercase								=		prepare_layoutlm_batch_inputs()

    # forward pass
    __lowercase								=		model(input_ids=UpperCAmelCase__,   bbox=UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__      )

    # test the sequence output on [0, :3, :3]
    __lowercase								=		torch.tensor(
        [[0.1_785, -0.1_947, -0.0_425], [-0.3_254, -0.2_807, 0.2_553], [-0.5_391, -0.3_322, 0.3_364]],   device=UpperCAmelCase__,   )

    self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3],   UpperCAmelCase__,   atol=1E-3      )      )

    # test the pooled output on [1, :3]
    __lowercase								=		torch.tensor([-0.6_580, -0.0_214, 0.8_552],   device=UpperCAmelCase__      )

    self.assertTrue(torch.allclose(outputs.pooler_output[1, :3],   UpperCAmelCase__,   atol=1E-3      )      )


 @slow
 def    _lowercase					(     self				:     str      ):
    # initialize model with randomly initialized sequence classification head
    __lowercase								=		LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased",   num_labels=2      ).to(
        UpperCAmelCase__      )

    __lowercase						,__lowercase						,__lowercase						,__lowercase						,__lowercase								=		prepare_layoutlm_batch_inputs()

    # forward pass
    __lowercase								=		model(
        input_ids=UpperCAmelCase__,   bbox=UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   labels=torch.tensor([1, 1],   device=UpperCAmelCase__      ),   )

    # test whether we get a loss as a scalar
    __lowercase								=		outputs.loss
    __lowercase								=		torch.Size([]      )
    self.assertEqual(loss.shape,   UpperCAmelCase__      )

    # test the shape of the logits
    __lowercase								=		outputs.logits
    __lowercase								=		torch.Size((2, 2)      )
    self.assertEqual(logits.shape,   UpperCAmelCase__      )


 @slow
 def    _lowercase					(     self				:     List[str]      ):
    # initialize model with randomly initialized token classification head
    __lowercase								=		LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased",   num_labels=1_3      ).to(
        UpperCAmelCase__      )

    __lowercase						,__lowercase						,__lowercase						,__lowercase						,__lowercase								=		prepare_layoutlm_batch_inputs()

    # forward pass
    __lowercase								=		model(
        input_ids=UpperCAmelCase__,   bbox=UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__,   labels=UpperCAmelCase__      )

    # test the loss calculation to be around 2.65
    # expected_loss = torch.tensor(2.65, device=torch_device)

    # The loss is currently somewhat random and can vary between 0.1-0.3 atol.
    # self.assertTrue(torch.allclose(outputs.loss, expected_loss, atol=0.1))

    # test the shape of the logits
    __lowercase								=		outputs.logits
    __lowercase								=		torch.Size((2, 2_5, 1_3)      )
    self.assertEqual(logits.shape,   UpperCAmelCase__      )



 @slow
 def    _lowercase					(     self				:     Dict      ):
    # initialize model with randomly initialized token classification head
    __lowercase								=		LayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased"      ).to(UpperCAmelCase__      )

    __lowercase						,__lowercase						,__lowercase						,__lowercase						,__lowercase								=		prepare_layoutlm_batch_inputs()

    # forward pass
    __lowercase								=		model(input_ids=UpperCAmelCase__,   bbox=UpperCAmelCase__,   attention_mask=UpperCAmelCase__,   token_type_ids=UpperCAmelCase__      )

    # test the shape of the logits
    __lowercase								=		torch.Size((2, 2_5)      )
    self.assertEqual(outputs.start_logits.shape,   UpperCAmelCase__      )
    self.assertEqual(outputs.end_logits.shape,   UpperCAmelCase__      )




