





'''simple docstring'''

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (   snake_case__							,unittest.TestCase		):







					"""simple docstring"""






					SCREAMING_SNAKE_CASE							=	RealmTokenizer
					SCREAMING_SNAKE_CASE							=	RealmTokenizerFast
					SCREAMING_SNAKE_CASE							=	True
					SCREAMING_SNAKE_CASE							=	True
					SCREAMING_SNAKE_CASE							=	filter_non_english
					def 		lowerCamelCase_   (   self       :      Optional[Any]		):







							"""simple docstring"""
							super().setUp()

							__UpperCAmelCase	:							Union[str, Any]     =     [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							__UpperCAmelCase	:							List[str]     =     os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["vocab_file"]		)
							with open(self.vocab_file		,	"w"		,	encoding="utf-8"		) as vocab_writer:
									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
					def 		lowerCamelCase_   (   self       :      Optional[int]		,	UpperCAmelCase_       :      Dict		):







							"""simple docstring"""
							__UpperCAmelCase	:							Optional[Any]     =     "UNwant\u00E9d,running"
							__UpperCAmelCase	:							Tuple     =     "unwanted, running"
							return input_text, output_text
					def 		lowerCamelCase_   (   self       :      List[str]		):







							"""simple docstring"""
							__UpperCAmelCase	:							Optional[int]     =     self.tokenizer_class(self.vocab_file		)

							__UpperCAmelCase	:							List[str]     =     tokenizer.tokenize("UNwant\u00E9d,running"		)
							self.assertListEqual(UpperCAmelCase_		,	["un", "##want", "##ed", ",", "runn", "##ing"]		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_		)		,	[9, 6, 7, 12, 10, 11]		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							if not self.test_rust_tokenizer:
									return

							__UpperCAmelCase	:							Tuple     =     self.get_tokenizer()
							__UpperCAmelCase	:							Any     =     self.get_rust_tokenizer()

							__UpperCAmelCase	:							Optional[Any]     =     "UNwant\u00E9d,running"

							__UpperCAmelCase	:							Optional[int]     =     tokenizer.tokenize(UpperCAmelCase_		)
							__UpperCAmelCase	:							Optional[Any]     =     rust_tokenizer.tokenize(UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)

							__UpperCAmelCase	:							List[str]     =     tokenizer.encode(UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_		)
							__UpperCAmelCase	:							Dict     =     rust_tokenizer.encode(UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)

							__UpperCAmelCase	:							int     =     self.get_rust_tokenizer()
							__UpperCAmelCase	:							int     =     tokenizer.encode(UpperCAmelCase_		)
							__UpperCAmelCase	:							Union[str, Any]     =     rust_tokenizer.encode(UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)

							# With lower casing
							__UpperCAmelCase	:							Union[str, Any]     =     self.get_tokenizer(do_lower_case=UpperCAmelCase_		)
							__UpperCAmelCase	:							List[str]     =     self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_		)

							__UpperCAmelCase	:							Dict     =     "UNwant\u00E9d,running"

							__UpperCAmelCase	:							Tuple     =     tokenizer.tokenize(UpperCAmelCase_		)
							__UpperCAmelCase	:							int     =     rust_tokenizer.tokenize(UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)

							__UpperCAmelCase	:							Optional[int]     =     tokenizer.encode(UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_		)
							__UpperCAmelCase	:							List[str]     =     rust_tokenizer.encode(UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)

							__UpperCAmelCase	:							Optional[Any]     =     self.get_rust_tokenizer()
							__UpperCAmelCase	:							List[Any]     =     tokenizer.encode(UpperCAmelCase_		)
							__UpperCAmelCase	:							Any     =     rust_tokenizer.encode(UpperCAmelCase_		)
							self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							__UpperCAmelCase	:							Any     =     BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)		,	["ah", "\u535A", "\u63A8", "zz"]		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							__UpperCAmelCase	:							Optional[int]     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)		,	["hello", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,	["hello"]		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							__UpperCAmelCase	:							Optional[int]     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		,	strip_accents=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,	["hällo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,	["h\u00E9llo"]		)
					def 		lowerCamelCase_   (   self       :      List[str]		):







							"""simple docstring"""
							__UpperCAmelCase	:							Any     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		,	strip_accents=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,	["hallo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,	["hello"]		)
					def 		lowerCamelCase_   (   self       :      Tuple		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[str]     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,	["hallo", "!", "how", "are", "you", "?"]		)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,	["hello"]		)
					def 		lowerCamelCase_   (   self       :      str		):







							"""simple docstring"""
							__UpperCAmelCase	:							Any     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)		,	["HeLLo", "!", "how", "Are", "yoU", "?"]		)
					def 		lowerCamelCase_   (   self       :      Dict		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[Any]     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		,	strip_accents=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,	["HäLLo", "!", "how", "Are", "yoU", "?"]		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							__UpperCAmelCase	:							int     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		,	strip_accents=UpperCAmelCase_		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,	["HaLLo", "!", "how", "Are", "yoU", "?"]		)
					def 		lowerCamelCase_   (   self       :      Any		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[str]     =     BasicTokenizer(do_lower_case=UpperCAmelCase_		,	never_split=["[UNK]"]		)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)		,	["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)
					def 		lowerCamelCase_   (   self       :      Any		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[Any]     =     ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							__UpperCAmelCase	:							Optional[Any]     =     {}
							for i, token in enumerate(UpperCAmelCase_		):
									__UpperCAmelCase	:							Any     =     i
							__UpperCAmelCase	:							Dict     =     WordpieceTokenizer(vocab=UpperCAmelCase_		,	unk_token="[UNK]"		)

							self.assertListEqual(tokenizer.tokenize(""		)		,	[]		)

							self.assertListEqual(tokenizer.tokenize("unwanted running"		)		,	["un", "##want", "##ed", "runn", "##ing"]		)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"		)		,	["[UNK]", "runn", "##ing"]		)
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							self.assertTrue(_is_whitespace(" "		)		)
							self.assertTrue(_is_whitespace("\t"		)		)
							self.assertTrue(_is_whitespace("\r"		)		)
							self.assertTrue(_is_whitespace("\n"		)		)
							self.assertTrue(_is_whitespace("\u00A0"		)		)

							self.assertFalse(_is_whitespace("A"		)		)
							self.assertFalse(_is_whitespace("-"		)		)
					def 		lowerCamelCase_   (   self       :      Optional[Any]		):







							"""simple docstring"""
							self.assertTrue(_is_control("\u0005"		)		)

							self.assertFalse(_is_control("A"		)		)
							self.assertFalse(_is_control(" "		)		)
							self.assertFalse(_is_control("\t"		)		)
							self.assertFalse(_is_control("\r"		)		)
					def 		lowerCamelCase_   (   self       :      List[str]		):







							"""simple docstring"""
							self.assertTrue(_is_punctuation("-"		)		)
							self.assertTrue(_is_punctuation("$"		)		)
							self.assertTrue(_is_punctuation("`"		)		)
							self.assertTrue(_is_punctuation("."		)		)

							self.assertFalse(_is_punctuation("A"		)		)
							self.assertFalse(_is_punctuation(" "		)		)
					def 		lowerCamelCase_   (   self       :      Dict		):







							"""simple docstring"""
							__UpperCAmelCase	:							List[Any]     =     self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_		) for t in ["Test", "\xad", "test"]]		,	[["[UNK]"], [], ["[UNK]"]]		)

							if self.test_rust_tokenizer:
									__UpperCAmelCase	:							Optional[int]     =     self.get_rust_tokenizer()
									self.assertListEqual(
									    [rust_tokenizer.tokenize(UpperCAmelCase_		) for t in ["Test", "\xad", "test"]]		,	[["[UNK]"], [], ["[UNK]"]]		)
					@slow
					def 		lowerCamelCase_   (   self       :      Union[str, Any]		):







							"""simple docstring"""
							__UpperCAmelCase	:							int     =     self.tokenizer_class.from_pretrained("bert-base-uncased"		)

							__UpperCAmelCase	:							int     =     tokenizer.encode("sequence builders"		,	add_special_tokens=UpperCAmelCase_		)
							__UpperCAmelCase	:							Union[str, Any]     =     tokenizer.encode("multi-sequence build"		,	add_special_tokens=UpperCAmelCase_		)

							__UpperCAmelCase	:							int     =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_		)
							__UpperCAmelCase	:							Any     =     tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_		,	UpperCAmelCase_		)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]
					def 		lowerCamelCase_   (   self       :      str		):







							"""simple docstring"""
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
											__UpperCAmelCase	:							Tuple     =     self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_		,	**UpperCAmelCase_		)

											__UpperCAmelCase	:							Dict     =     f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
											__UpperCAmelCase	:							int     =     tokenizer_r.encode_plus(
											    UpperCAmelCase_		,	return_attention_mask=UpperCAmelCase_		,	return_token_type_ids=UpperCAmelCase_		,	return_offsets_mapping=UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_		,	)

											__UpperCAmelCase	:							str     =     tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_		,	"do_lower_case"		) else False
											__UpperCAmelCase	:							Union[str, Any]     =     (
											    [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "A"),
											        ((1, 2), ","),
											        ((3, 5), "na"),
											        ((5, 6), "##ï"),
											        ((6, 8), "##ve"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "Allen"),
											        ((21, 23), "##NL"),
											        ((23, 24), "##P"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											    if not do_lower_case
											    else [
											        ((0, 0), tokenizer_r.cls_token),
											        ((0, 1), "a"),
											        ((1, 2), ","),
											        ((3, 8), "naive"),
											        ((9, 15), tokenizer_r.mask_token),
											        ((16, 21), "allen"),
											        ((21, 23), "##nl"),
											        ((23, 24), "##p"),
											        ((25, 33), "sentence"),
											        ((33, 34), "."),
											        ((0, 0), tokenizer_r.sep_token),
											    ]
											)

											self.assertEqual(
											    [e[1] for e in expected_results]		,	tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
											self.assertEqual([e[0] for e in expected_results]		,	tokens["offset_mapping"]		)






					@slow
					def 		lowerCamelCase_   (   self       :      int		):







							"""simple docstring"""
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
											__UpperCAmelCase	:							Dict     =     self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_		,	**UpperCAmelCase_		)
											__UpperCAmelCase	:							int     =     self.tokenizer_class.from_pretrained(UpperCAmelCase_		,	**UpperCAmelCase_		)
											__UpperCAmelCase	:							Dict     =     [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

											__UpperCAmelCase	:							Union[str, Any]     =     tokenizer_r.batch_encode_candidates(UpperCAmelCase_		,	max_length=10		,	return_tensors="np"		)
											__UpperCAmelCase	:							Optional[int]     =     tokenizer_p.batch_encode_candidates(UpperCAmelCase_		,	max_length=10		,	return_tensors="np"		)

											__UpperCAmelCase	:							Optional[int]     =     (2, 2, 10)

											self.assertEqual(encoded_sentence_r["input_ids"].shape		,	UpperCAmelCase_		)
											self.assertEqual(encoded_sentence_r["attention_mask"].shape		,	UpperCAmelCase_		)
											self.assertEqual(encoded_sentence_r["token_type_ids"].shape		,	UpperCAmelCase_		)

											self.assertEqual(encoded_sentence_p["input_ids"].shape		,	UpperCAmelCase_		)
											self.assertEqual(encoded_sentence_p["attention_mask"].shape		,	UpperCAmelCase_		)
											self.assertEqual(encoded_sentence_p["token_type_ids"].shape		,	UpperCAmelCase_		)











'''simple docstring'''

from typing import List

import PIL
import torch
from PIL import Image

from ...configuration_utils import ConfigMixin
from ...models.modeling_utils import ModelMixin
from ...utils import PIL_INTERPOLATION


class      SCREAMING_SNAKE_CASE__      (   snake_case__							,snake_case__		):







					"""simple docstring"""






					def __init__(   self       :      Any		):







							"""simple docstring"""
							super().__init__()

							self.register_buffer("watermark_image"		,	torch.zeros((62, 62, 4)		)		)
							__UpperCAmelCase	:							Optional[int]     =     None






					def 		lowerCamelCase_   (   self       :      List[Any]		,	UpperCAmelCase_       :      List[PIL.Image.Image]		,	UpperCAmelCase_       :      int=None		):







							"""simple docstring"""
							# copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287

							__UpperCAmelCase	:							int     =     images[0].height
							__UpperCAmelCase	:							Any     =     images[0].width

							__UpperCAmelCase	:							Union[str, Any]     =     sample_size or h

							__UpperCAmelCase	:							Dict     =     min(h / sample_size		,	w / sample_size		)
							__UpperCAmelCase  ,			__UpperCAmelCase	:							Optional[int]     =     (int(h / coef		), int(w / coef		)) if coef < 1 else (h, w)

							__UpperCAmelCase  ,			__UpperCAmelCase	:							Union[str, Any]     =     1_024**2, img_w * img_h
							__UpperCAmelCase	:							Optional[int]     =     (Sa / Sa) ** 0.5
							__UpperCAmelCase  ,			__UpperCAmelCase  ,			__UpperCAmelCase	:							str     =     int(K * 62		), img_w - int(14 * K		), img_h - int(14 * K		)

							if self.watermark_image_as_pil is None:
									__UpperCAmelCase	:							int     =     self.watermark_image.to(torch.uinta		).cpu().numpy()
									__UpperCAmelCase	:							Optional[int]     =     Image.fromarray(UpperCAmelCase_		,	mode="RGBA"		)
									__UpperCAmelCase	:							Optional[int]     =     watermark_image

							__UpperCAmelCase	:							List[str]     =     self.watermark_image_as_pil.resize(
							    (wm_size, wm_size)		,	PIL_INTERPOLATION["bicubic"]		,	reducing_gap=UpperCAmelCase_		)

							for pil_img in images:
									pil_img.paste(UpperCAmelCase_		,	box=(wm_x - wm_size, wm_y - wm_size, wm_x, wm_y)		,	mask=wm_img.split()[-1]		)

							return images




