
"""simple docstring"""






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class      _SCREAMING_SNAKE_CASE   (     A__      ,					unittest.TestCase				):
			UpperCAmelCase_    :Dict												=		ElectraTokenizer
			UpperCAmelCase_    :List[str]												=		ElectraTokenizerFast
			UpperCAmelCase_    :Dict												=		True
			UpperCAmelCase_    :List[str]												=		True
			UpperCAmelCase_    :Optional[int]												=		filter_non_english






			def 					__lowerCAmelCase					(   self			)     ->	List[Any]:
						super().setUp()

						lowerCAmelCase_       :Optional[int]     	=    [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """want""",
						    """##want""",
						    """##ed""",
						    """wa""",
						    """un""",
						    """runn""",
						    """##ing""",
						    """,""",
						    """low""",
						    """lowest""",
						]
						lowerCAmelCase_       :Any     	=    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["""vocab_file"""]			)
						with open(self.vocab_file			,  """w"""			,  encoding="""utf-8"""			) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[Any]:
						lowerCAmelCase_       :List[str]     	=    """UNwant\u00E9d,running"""
						lowerCAmelCase_       :List[Any]     	=    """unwanted, running"""
						return input_text, output_text






			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer_class(self.vocab_file			)

						lowerCAmelCase_       :Dict     	=    tokenizer.tokenize("""UNwant\u00E9d,running"""			)
						self.assertListEqual(__A			,  ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__A			)			,  [9, 6, 7, 12, 10, 11]			)






			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						if not self.test_rust_tokenizer:
									return

						lowerCAmelCase_       :Optional[int]     	=    self.get_tokenizer()
						lowerCAmelCase_       :int     	=    self.get_rust_tokenizer()

						lowerCAmelCase_       :List[str]     	=    """UNwant\u00E9d,running"""

						lowerCAmelCase_       :List[Any]     	=    tokenizer.tokenize(__A			)
						lowerCAmelCase_       :str     	=    rust_tokenizer.tokenize(__A			)
						self.assertListEqual(__A			,  __A			)

						lowerCAmelCase_       :Any     	=    tokenizer.encode(__A			,  add_special_tokens=__A			)
						lowerCAmelCase_       :Optional[int]     	=    rust_tokenizer.encode(__A			,  add_special_tokens=__A			)
						self.assertListEqual(__A			,  __A			)

						lowerCAmelCase_       :Optional[int]     	=    self.get_rust_tokenizer()
						lowerCAmelCase_       :str     	=    tokenizer.encode(__A			)
						lowerCAmelCase_       :Optional[int]     	=    rust_tokenizer.encode(__A			)
						self.assertListEqual(__A			,  __A			)

						# With lower casing
						lowerCAmelCase_       :List[Any]     	=    self.get_tokenizer(do_lower_case=__A			)
						lowerCAmelCase_       :Optional[Any]     	=    self.get_rust_tokenizer(do_lower_case=__A			)

						lowerCAmelCase_       :Tuple     	=    """UNwant\u00E9d,running"""

						lowerCAmelCase_       :Any     	=    tokenizer.tokenize(__A			)
						lowerCAmelCase_       :Any     	=    rust_tokenizer.tokenize(__A			)
						self.assertListEqual(__A			,  __A			)

						lowerCAmelCase_       :Any     	=    tokenizer.encode(__A			,  add_special_tokens=__A			)
						lowerCAmelCase_       :List[Any]     	=    rust_tokenizer.encode(__A			,  add_special_tokens=__A			)
						self.assertListEqual(__A			,  __A			)

						lowerCAmelCase_       :Any     	=    self.get_rust_tokenizer()
						lowerCAmelCase_       :Optional[Any]     	=    tokenizer.encode(__A			)
						lowerCAmelCase_       :Tuple     	=    rust_tokenizer.encode(__A			)
						self.assertListEqual(__A			,  __A			)






			def 					__lowerCAmelCase					(   self			)     ->	List[Any]:
						lowerCAmelCase_       :Optional[Any]     	=    BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			)			,  ["""ah""", """\u535A""", """\u63A8""", """zz"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :Optional[Any]     	=    BasicTokenizer(do_lower_case=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)			,  ["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)			,  ["""hello"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	List[str]:
						lowerCAmelCase_       :Optional[int]     	=    BasicTokenizer(do_lower_case=__A			,  strip_accents=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)			,  ["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)			,  ["""h\u00E9llo"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	int:
						lowerCAmelCase_       :Any     	=    BasicTokenizer(do_lower_case=__A			,  strip_accents=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)			,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)			,  ["""hello"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						lowerCAmelCase_       :List[Any]     	=    BasicTokenizer(do_lower_case=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)			,  ["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)			,  ["""hello"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	List[Any]:
						lowerCAmelCase_       :Tuple     	=    BasicTokenizer(do_lower_case=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)			,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :List[str]     	=    BasicTokenizer(do_lower_case=__A			,  strip_accents=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)			,  ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	str:
						lowerCAmelCase_       :Optional[int]     	=    BasicTokenizer(do_lower_case=__A			,  strip_accents=__A			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)			,  ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Union[str, Any]:
						lowerCAmelCase_       :Tuple     	=    BasicTokenizer(do_lower_case=__A			,  never_split=["""[UNK]"""]			)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			)			,  ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :List[Any]     	=    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						lowerCAmelCase_       :List[Any]     	=    {}
						for i, token in enumerate(__A			):
									lowerCAmelCase_       :str     	=    i
						lowerCAmelCase_       :Optional[int]     	=    WordpieceTokenizer(vocab=__A			,  unk_token="""[UNK]"""			)

						self.assertListEqual(tokenizer.tokenize(""""""			)			,  []			)

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""			)			,  ["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			)			,  ["""[UNK]""", """runn""", """##ing"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						self.assertTrue(_is_whitespace(""" """			)			)
						self.assertTrue(_is_whitespace("""\t"""			)			)
						self.assertTrue(_is_whitespace("""\r"""			)			)
						self.assertTrue(_is_whitespace("""\n"""			)			)
						self.assertTrue(_is_whitespace("""\u00A0"""			)			)

						self.assertFalse(_is_whitespace("""A"""			)			)
						self.assertFalse(_is_whitespace("""-"""			)			)






			def 					__lowerCAmelCase					(   self			)     ->	Union[str, Any]:
						self.assertTrue(_is_control("""\u0005"""			)			)

						self.assertFalse(_is_control("""A"""			)			)
						self.assertFalse(_is_control(""" """			)			)
						self.assertFalse(_is_control("""\t"""			)			)
						self.assertFalse(_is_control("""\r"""			)			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[int]:
						self.assertTrue(_is_punctuation("""-"""			)			)
						self.assertTrue(_is_punctuation("""$"""			)			)
						self.assertTrue(_is_punctuation("""`"""			)			)
						self.assertTrue(_is_punctuation("""."""			)			)

						self.assertFalse(_is_punctuation("""A"""			)			)
						self.assertFalse(_is_punctuation(""" """			)			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[int]:
						lowerCAmelCase_       :List[Any]     	=    self.get_tokenizer()
						lowerCAmelCase_       :Union[str, Any]     	=    self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__A			) for t in ["""Test""", """\xad""", """test"""]]			,  [["""[UNK]"""], [], ["""[UNK]"""]]			)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__A			) for t in ["""Test""", """\xad""", """test"""]]			,  [["""[UNK]"""], [], ["""[UNK]"""]]			)






			@slow
			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :List[Any]     	=    self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""			)

						lowerCAmelCase_       :str     	=    tokenizer.encode("""sequence builders"""			,  add_special_tokens=__A			)
						lowerCAmelCase_       :str     	=    tokenizer.encode("""multi-sequence build"""			,  add_special_tokens=__A			)

						lowerCAmelCase_       :List[Any]     	=    tokenizer.build_inputs_with_special_tokens(__A			)
						lowerCAmelCase_       :Dict     	=    tokenizer.build_inputs_with_special_tokens(__A			,  __A			)

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]






			def 					__lowerCAmelCase					(   self			)     ->	Union[str, Any]:
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
												lowerCAmelCase_       :Optional[int]     	=    self.rust_tokenizer_class.from_pretrained(__A			,  **__A			)

												lowerCAmelCase_       :str     	=    f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
												lowerCAmelCase_       :Any     	=    tokenizer_r.encode_plus(
												    __A			,  return_attention_mask=__A			,  return_token_type_ids=__A			,  return_offsets_mapping=__A			,  add_special_tokens=__A			,  )

												lowerCAmelCase_       :Any     	=    tokenizer_r.do_lower_case if hasattr(__A			,  """do_lower_case"""			) else False
												lowerCAmelCase_       :Dict     	=    (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), """A"""),
												        ((1, 2), ""","""),
												        ((3, 5), """na"""),
												        ((5, 6), """##ï"""),
												        ((6, 8), """##ve"""),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), """Allen"""),
												        ((21, 23), """##NL"""),
												        ((23, 24), """##P"""),
												        ((25, 33), """sentence"""),
												        ((33, 34), """."""),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), """a"""),
												        ((1, 2), ""","""),
												        ((3, 8), """naive"""),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), """allen"""),
												        ((21, 23), """##nl"""),
												        ((23, 24), """##p"""),
												        ((25, 33), """sentence"""),
												        ((33, 34), """."""),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]			)			)
												self.assertEqual([e[0] for e in expected_results]			,  tokens["""offset_mapping"""]			)






			def 					__lowerCAmelCase					(   self			)     ->	Tuple:
						lowerCAmelCase_       :str     	=    ["""的""", """人""", """有"""]
						lowerCAmelCase_       :List[str]     	=    """""".join(__A			)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
												lowerCAmelCase_       :Dict     	=    True
												lowerCAmelCase_       :List[Any]     	=    self.tokenizer_class.from_pretrained(__A			,  **__A			)
												lowerCAmelCase_       :Optional[Any]     	=    self.rust_tokenizer_class.from_pretrained(__A			,  **__A			)

												lowerCAmelCase_       :Tuple     	=    tokenizer_p.encode(__A			,  add_special_tokens=__A			)
												lowerCAmelCase_       :int     	=    tokenizer_r.encode(__A			,  add_special_tokens=__A			)

												lowerCAmelCase_       :Union[str, Any]     	=    tokenizer_r.convert_ids_to_tokens(__A			)
												lowerCAmelCase_       :Dict     	=    tokenizer_p.convert_ids_to_tokens(__A			)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(__A			,  __A			)
												self.assertListEqual(__A			,  __A			)

												lowerCAmelCase_       :List[str]     	=    False
												lowerCAmelCase_       :Tuple     	=    self.rust_tokenizer_class.from_pretrained(__A			,  **__A			)
												lowerCAmelCase_       :str     	=    self.tokenizer_class.from_pretrained(__A			,  **__A			)

												lowerCAmelCase_       :int     	=    tokenizer_r.encode(__A			,  add_special_tokens=__A			)
												lowerCAmelCase_       :List[str]     	=    tokenizer_p.encode(__A			,  add_special_tokens=__A			)

												lowerCAmelCase_       :Optional[Any]     	=    tokenizer_r.convert_ids_to_tokens(__A			)
												lowerCAmelCase_       :List[str]     	=    tokenizer_p.convert_ids_to_tokens(__A			)

												# it is expected that only the first Chinese character is not preceded by "##".
												lowerCAmelCase_       :Union[str, Any]     	=    [
												    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__A			)
												]
												self.assertListEqual(__A			,  __A			)
												self.assertListEqual(__A			,  __A			)








"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import is_detectrona_available, logging


__UpperCAmelCase 					=    logging.get_logger(__name__)

__UpperCAmelCase 					=    {
    'layoutlmv2-base-uncased': 'https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/config.json',
    'layoutlmv2-large-uncased': 'https://huggingface.co/microsoft/layoutlmv2-large-uncased/resolve/main/config.json',
    # See all LayoutLMv2 models at https://huggingface.co/models?filter=layoutlmv2
}

# soft dependency
if is_detectrona_available():
				import detectrona




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Optional[int]												=		"layoutlmv2"






			def __init__(   self			,  __A=3_0522			,  __A=768			,  __A=12			,  __A=12			,  __A=3072			,  __A="gelu"			,  __A=0.1			,  __A=0.1			,  __A=512			,  __A=2			,  __A=0.0_2			,  __A=1E-12			,  __A=0			,  __A=1024			,  __A=128			,  __A=32			,  __A=True			,  __A=256			,  __A=64			,  __A=True			,  __A=[7, 7, 256]			,  __A=128			,  __A=128			,  __A=True			,  __A=True			,  __A=False			,  __A=None			,  **__A			,  )     ->	List[str]:
						super().__init__(
						    vocab_size=__A			,  hidden_size=__A			,  num_hidden_layers=__A			,  num_attention_heads=__A			,  intermediate_size=__A			,  hidden_act=__A			,  hidden_dropout_prob=__A			,  attention_probs_dropout_prob=__A			,  max_position_embeddings=__A			,  type_vocab_size=__A			,  initializer_range=__A			,  layer_norm_eps=__A			,  pad_token_id=__A			,  **__A			,  )
						lowerCAmelCase_       :Optional[Any]     	=    max_ad_position_embeddings
						lowerCAmelCase_       :Dict     	=    max_rel_pos
						lowerCAmelCase_       :str     	=    rel_pos_bins
						lowerCAmelCase_       :List[str]     	=    fast_qkv
						lowerCAmelCase_       :Union[str, Any]     	=    max_rel_ad_pos
						lowerCAmelCase_       :List[str]     	=    rel_ad_pos_bins
						lowerCAmelCase_       :Any     	=    convert_sync_batchnorm
						lowerCAmelCase_       :Dict     	=    image_feature_pool_shape
						lowerCAmelCase_       :str     	=    coordinate_size
						lowerCAmelCase_       :Tuple     	=    shape_size
						lowerCAmelCase_       :Tuple     	=    has_relative_attention_bias
						lowerCAmelCase_       :Union[str, Any]     	=    has_spatial_attention_bias
						lowerCAmelCase_       :int     	=    has_visual_segment_embedding
						lowerCAmelCase_       :Any     	=    (
						    detectrona_config_args if detectrona_config_args is not None else self.get_default_detectrona_config()
						)






			@classmethod
			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						return {
						    "MODEL.MASK_ON": True,
						    "MODEL.PIXEL_STD": [5_7.3_7_5, 5_7.1_2_0, 5_8.3_9_5],
						    "MODEL.BACKBONE.NAME": "build_resnet_fpn_backbone",
						    "MODEL.FPN.IN_FEATURES": ["res2", "res3", "res4", "res5"],
						    "MODEL.ANCHOR_GENERATOR.SIZES": [[32], [64], [128], [256], [512]],
						    "MODEL.RPN.IN_FEATURES": ["p2", "p3", "p4", "p5", "p6"],
						    "MODEL.RPN.PRE_NMS_TOPK_TRAIN": 2000,
						    "MODEL.RPN.PRE_NMS_TOPK_TEST": 1000,
						    "MODEL.RPN.POST_NMS_TOPK_TRAIN": 1000,
						    "MODEL.POST_NMS_TOPK_TEST": 1000,
						    "MODEL.ROI_HEADS.NAME": "StandardROIHeads",
						    "MODEL.ROI_HEADS.NUM_CLASSES": 5,
						    "MODEL.ROI_HEADS.IN_FEATURES": ["p2", "p3", "p4", "p5"],
						    "MODEL.ROI_BOX_HEAD.NAME": "FastRCNNConvFCHead",
						    "MODEL.ROI_BOX_HEAD.NUM_FC": 2,
						    "MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION": 14,
						    "MODEL.ROI_MASK_HEAD.NAME": "MaskRCNNConvUpsampleHead",
						    "MODEL.ROI_MASK_HEAD.NUM_CONV": 4,
						    "MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION": 7,
						    "MODEL.RESNETS.DEPTH": 101,
						    "MODEL.RESNETS.SIZES": [[32], [64], [128], [256], [512]],
						    "MODEL.RESNETS.ASPECT_RATIOS": [[0.5, 1.0, 2.0]],
						    "MODEL.RESNETS.OUT_FEATURES": ["res2", "res3", "res4", "res5"],
						    "MODEL.RESNETS.NUM_GROUPS": 32,
						    "MODEL.RESNETS.WIDTH_PER_GROUP": 8,
						    "MODEL.RESNETS.STRIDE_IN_1X1": False,
						}






			def 					__lowerCAmelCase					(   self			)     ->	List[Any]:
						lowerCAmelCase_       :Any     	=    detectrona.config.get_cfg()
						for k, v in self.detectrona_config_args.items():
									lowerCAmelCase_       :int     	=    k.split("""."""			)
									lowerCAmelCase_       :List[Any]     	=    detectrona_config
									for attribute in attributes[:-1]:
												lowerCAmelCase_       :List[Any]     	=    getattr(__A			,  __A			)
									setattr(__A			,  attributes[-1]			,  __A			)

						return detectrona_config






