


'''simple docstring'''



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 							__UpperCamelCase				(		lowerCAmelCase_		,      unittest.TestCase					):
	A_				=    BertTokenizer
	A_				=    BertTokenizerFast
	A_				=    True
	A_				=    True
	A_				=    filter_non_english




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						super().setUp()

						__a			:       str					    =  [
						    '[UNK]',
						    '[CLS]',
						    '[SEP]',
						    '[PAD]',
						    '[MASK]',
						    'want',
						    '##want',
						    '##ed',
						    'wa',
						    'un',
						    'runn',
						    '##ing',
						    ',',
						    'low',
						    'lowest',
						]
						__a			:       int					    =  os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['vocab_file']							)
						with open(self.vocab_file						,				'w'						,				encoding='utf-8'							) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						__a			:       int					    =  'UNwant\u00E9d,running'
						__a			:       int					    =  'unwanted, running'
						return input_text, output_text




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Any					    =  self.tokenizer_class(self.vocab_file							)

						__a			:       Optional[Any]					    =  tokenizer.tokenize('UNwant\u00E9d,running'							)
						self.assertListEqual(__a						,				['un', '##want', '##ed', ',', 'runn', '##ing']							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,				[9, 6, 7, 12, 10, 11]							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						if not self.test_rust_tokenizer:
											return

						__a			:       Optional[Any]					    =  self.get_tokenizer()
						__a			:       Any					    =  self.get_rust_tokenizer()

						__a			:       int					    =  'UNwant\u00E9d,running'

						__a			:       Tuple					    =  tokenizer.tokenize(__a							)
						__a			:       List[str]					    =  rust_tokenizer.tokenize(__a							)
						self.assertListEqual(__a						,				__a							)

						__a			:       Any					    =  tokenizer.encode(__a						,				add_special_tokens=__a							)
						__a			:       Optional[int]					    =  rust_tokenizer.encode(__a						,				add_special_tokens=__a							)
						self.assertListEqual(__a						,				__a							)

						__a			:       Optional[Any]					    =  self.get_rust_tokenizer()
						__a			:       Optional[int]					    =  tokenizer.encode(__a							)
						__a			:       int					    =  rust_tokenizer.encode(__a							)
						self.assertListEqual(__a						,				__a							)

						# With lower casing
						__a			:       Dict					    =  self.get_tokenizer(do_lower_case=__a							)
						__a			:       Optional[int]					    =  self.get_rust_tokenizer(do_lower_case=__a							)

						__a			:       Dict					    =  'UNwant\u00E9d,running'

						__a			:       int					    =  tokenizer.tokenize(__a							)
						__a			:       int					    =  rust_tokenizer.tokenize(__a							)
						self.assertListEqual(__a						,				__a							)

						__a			:       Dict					    =  tokenizer.encode(__a						,				add_special_tokens=__a							)
						__a			:       str					    =  rust_tokenizer.encode(__a						,				add_special_tokens=__a							)
						self.assertListEqual(__a						,				__a							)

						__a			:       List[str]					    =  self.get_rust_tokenizer()
						__a			:       List[Any]					    =  tokenizer.encode(__a							)
						__a			:       Tuple					    =  rust_tokenizer.encode(__a							)
						self.assertListEqual(__a						,				__a							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)						,				['ah', '\u535A', '\u63A8', 'zz']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Any					    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,				['hello', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  BasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hällo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['h\u00E9llo']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       int					    =  BasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,				['hello']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       str					    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[str]					    =  BasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  BasicTokenizer(do_lower_case=__a						,				strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[Any]					    =  BasicTokenizer(do_lower_case=__a						,				never_split=['[UNK]']							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)						,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  BasicTokenizer()
						__a			:       List[str]					    =  'a\n\'ll !!to?\'d of, can\'t.'
						__a			:       Union[str, Any]					    =  ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
						self.assertListEqual(tokenizer.tokenize(__a							)						,				__a							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__a			:       Dict					    =  {}
						for i, token in enumerate(__a							):
											__a			:       Tuple					    =  i
						__a			:       Union[str, Any]					    =  WordpieceTokenizer(vocab=__a						,				unk_token='[UNK]'							)

						self.assertListEqual(tokenizer.tokenize(''							)						,				[]							)

						self.assertListEqual(tokenizer.tokenize('unwanted running'							)						,				['un', '##want', '##ed', 'runn', '##ing']							)

						self.assertListEqual(tokenizer.tokenize('unwantedX running'							)						,				['[UNK]', 'runn', '##ing']							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_whitespace(' '							)							)
						self.assertTrue(_is_whitespace('\t'							)							)
						self.assertTrue(_is_whitespace('\r'							)							)
						self.assertTrue(_is_whitespace('\n'							)							)
						self.assertTrue(_is_whitespace('\u00A0'							)							)

						self.assertFalse(_is_whitespace('A'							)							)
						self.assertFalse(_is_whitespace('-'							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_control('\u0005'							)							)

						self.assertFalse(_is_control('A'							)							)
						self.assertFalse(_is_control(' '							)							)
						self.assertFalse(_is_control('\t'							)							)
						self.assertFalse(_is_control('\r'							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						self.assertTrue(_is_punctuation('-'							)							)
						self.assertTrue(_is_punctuation('$'							)							)
						self.assertTrue(_is_punctuation('`'							)							)
						self.assertTrue(_is_punctuation('.'							)							)

						self.assertFalse(_is_punctuation('A'							)							)
						self.assertFalse(_is_punctuation(' '							)							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  self.get_tokenizer()
						__a			:       int					    =  self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__a							) for t in ['Test', '\xad', 'test']]						,				[['[UNK]'], [], ['[UNK]']]							)

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__a							) for t in ['Test', '\xad', 'test']]						,				[['[UNK]'], [], ['[UNK]']]							)




	@slow
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  self.tokenizer_class.from_pretrained('bert-base-uncased'							)

						__a			:       Optional[int]					    =  tokenizer.encode('sequence builders'						,				add_special_tokens=__a							)
						__a			:       Any					    =  tokenizer.encode('multi-sequence build'						,				add_special_tokens=__a							)

						__a			:       List[str]					    =  tokenizer.build_inputs_with_special_tokens(__a							)
						__a			:       Any					    =  tokenizer.build_inputs_with_special_tokens(__a						,				__a							)

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a			:       str					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       Dict					    =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																__a			:       Optional[int]					    =  tokenizer_r.encode_plus(
																    __a						,				return_attention_mask=__a						,				return_token_type_ids=__a						,				return_offsets_mapping=__a						,				add_special_tokens=__a						,				)

																__a			:       Optional[int]					    =  tokenizer_r.do_lower_case if hasattr(__a						,				'do_lower_case'							) else False
																__a			:       Dict					    =  (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'A'),
																        ((1, 2), ','),
																        ((3, 5), 'na'),
																        ((5, 6), '##ï'),
																        ((6, 8), '##ve'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'Allen'),
																        ((21, 23), '##NL'),
																        ((23, 24), '##P'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), 'a'),
																        ((1, 2), ','),
																        ((3, 8), 'naive'),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), 'allen'),
																        ((21, 23), '##nl'),
																        ((23, 24), '##p'),
																        ((25, 33), 'sentence'),
																        ((33, 34), '.'),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]						,				tokenizer_r.convert_ids_to_tokens(tokens['input_ids']							)							)
																self.assertEqual([e[0] for e in expected_results]						,				tokens['offset_mapping']							)





	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Union[str, Any]					    =  ['的', '人', '有']
						__a			:       List[str]					    =  ''.join(__a							)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																__a			:       Any					    =  True
																__a			:       Any					    =  self.tokenizer_class.from_pretrained(__a						,				**__a							)
																__a			:       Any					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       Tuple					    =  tokenizer_p.encode(__a						,				add_special_tokens=__a							)
																__a			:       Union[str, Any]					    =  tokenizer_r.encode(__a						,				add_special_tokens=__a							)

																__a			:       Dict					    =  tokenizer_r.convert_ids_to_tokens(__a							)
																__a			:       Optional[Any]					    =  tokenizer_p.convert_ids_to_tokens(__a							)

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(__a						,				__a							)
																self.assertListEqual(__a						,				__a							)

																__a			:       Union[str, Any]					    =  False
																__a			:       List[str]					    =  self.rust_tokenizer_class.from_pretrained(__a						,				**__a							)
																__a			:       Optional[int]					    =  self.tokenizer_class.from_pretrained(__a						,				**__a							)

																__a			:       List[str]					    =  tokenizer_r.encode(__a						,				add_special_tokens=__a							)
																__a			:       List[str]					    =  tokenizer_p.encode(__a						,				add_special_tokens=__a							)

																__a			:       str					    =  tokenizer_r.convert_ids_to_tokens(__a							)
																__a			:       Union[str, Any]					    =  tokenizer_p.convert_ids_to_tokens(__a							)

																# it is expected that only the first Chinese character is not preceded by "##".
																__a			:       Optional[Any]					    =  [
																    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__a							)
																]
																self.assertListEqual(__a						,				__a							)
																self.assertListEqual(__a						,				__a							)







'''simple docstring'''



import unittest

import numpy as np

from diffusers import OnnxStableDiffusionInpaintPipelineLegacy
from diffusers.utils.testing_utils import (
    is_onnx_available,
    load_image,
    load_numpy,
    nightly,
    require_onnxruntime,
    require_torch_gpu,
)


if is_onnx_available():
	import onnxruntime as ort





@nightly
@require_onnxruntime
@require_torch_gpu
class 							__UpperCamelCase				(		unittest.TestCase					):




	@property
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						return (
						    "CUDAExecutionProvider",
						    {
						        "gpu_mem_limit": "15000000000",  # 15GB
						        "arena_extend_strategy": "kSameAsRequested",
						    },
						)




	@property
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Dict					    =  ort.SessionOptions()
						__a			:       Dict					    =  False
						return options





	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Tuple					    =  load_image(
						    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
						    '/in_paint/overture-creations-5sI6fQgYIuo.png'							)
						__a			:       int					    =  load_image(
						    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
						    '/in_paint/overture-creations-5sI6fQgYIuo_mask.png'							)
						__a			:       Dict					    =  load_numpy(
						    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
						    '/in_paint/red_cat_sitting_on_a_park_bench_onnx.npy'							)

						# using the PNDM scheduler by default
						__a			:       str					    =  OnnxStableDiffusionInpaintPipelineLegacy.from_pretrained(
						    'CompVis/stable-diffusion-v1-4'						,				revision='onnx'						,				safety_checker=__a						,				feature_extractor=__a						,				provider=self.gpu_provider						,				sess_options=self.gpu_options						,				)
						pipe.set_progress_bar_config(disable=__a							)

						__a			:       Tuple					    =  'A red cat sitting on a park bench'

						__a			:       int					    =  np.random.RandomState(0							)
						__a			:       Tuple					    =  pipe(
						    prompt=__a						,				image=__a						,				mask_image=__a						,				strength=0.75						,				guidance_scale=7.5						,				num_inference_steps=15						,				generator=__a						,				output_type='np'						,				)

						__a			:       Tuple					    =  output.images[0]

						assert image.shape == (512, 512, 3)
						assert np.abs(expected_image - image							).max() < 1E-2



