



import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case		=  get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")



@require_sentencepiece
@require_tokenizers
class   UpperCAmelCase_    (       a   ,					unittest.TestCase):
							lowerCamelCase__   			=		GPTSwaTokenizer
							lowerCamelCase__   			=		False
							lowerCamelCase__   			=		True
							lowerCamelCase__   			=		False







							def    snake_case__      (   self):

														'''simple docstring'''

														super().setUp()

														# We have a SentencePiece fixture for testing
														_lowerCAmelCase     :							Union[str, Any]    =    GPTSwaTokenizer(__a,			eos_token="<unk>",			bos_token="<unk>",			pad_token="<unk>")

														tokenizer.save_pretrained(self.tmpdirname)







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							List[str]    =    "This is a test"
														_lowerCAmelCase     :							List[Any]    =    "This is a test"
														return input_text, output_text







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    "<s>"
														_lowerCAmelCase     :							List[Any]    =    1

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a),			__a)
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a),			__a)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    list(self.get_tokenizer().get_vocab().keys())

														self.assertEqual(vocab_keys[0],			"<unk>")
														self.assertEqual(vocab_keys[1],			"<s>")
														self.assertEqual(vocab_keys[-1],			"j")
														self.assertEqual(len(__a),			2000)







							def    snake_case__      (   self):

														'''simple docstring'''

														self.assertEqual(self.get_tokenizer().vocab_size,			2000)







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Dict    =    GPTSwaTokenizer(__a)

														_lowerCAmelCase     :							Optional[int]    =    tokenizer.tokenize("This is a test")
														self.assertListEqual(__a,			["▁This", "▁is", "▁a", "▁t", "est"])

														self.assertListEqual(tokenizer.convert_tokens_to_ids(__a),			[465, 287, 265, 631, 842])

														_lowerCAmelCase     :							Optional[Any]    =    tokenizer.tokenize("I was born in 92000, and this is falsé.")
														# fmt: off
														self.assertListEqual(
														    __a,			["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."],			)
														# fmt: on

														_lowerCAmelCase     :							str    =    tokenizer.convert_tokens_to_ids(__a)
														self.assertListEqual(
														    __a,			[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],			)

														_lowerCAmelCase     :							Any    =    tokenizer.convert_ids_to_tokens(__a)
														# fmt: off
														self.assertListEqual(
														    __a,			["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."])
														# fmt: on







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    GPTSwaTokenizer(__a)
														_lowerCAmelCase     :							Optional[int]    =    ["This is a test", "I was born in 92000, and this is falsé."]
														_lowerCAmelCase     :							Optional[Any]    =    [
														    [465, 287, 265, 631, 842],
														    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
														]

														# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
														for text, expected_ids in zip(__a,			__a):
																					self.assertListEqual(tokenizer.encode_fast(__a),			__a)

														# Test that decode_fast returns the input text
														for text, token_ids in zip(__a,			__a):
																					self.assertEqual(tokenizer.decode_fast(__a),			__a)







							@slow
							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    [
														    "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
														    "Hey there, how are you doing this fine day?",
														    "This is a text with a trailing spaces followed by a dot     .",
														    "Häj sväjs lillebrör! =)",
														    "Det är inget fel på Mr. Cool",
														]

														# fmt: off
														_lowerCAmelCase     :							Union[str, Any]    =    {"input_ids": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
														# fmt: on
														self.tokenizer_integration_test_util(
														    expected_encoding=__a,			model_name="AI-Sweden/gpt-sw3-126m",			sequences=__a,			)





import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
						import torch

if is_vision_available():
						import PIL


_snake_case		=  logging.get_logger(__name__)






def 				A				(					_lowerCamelCase				, _lowerCamelCase				, _lowerCamelCase				, _lowerCamelCase  ):





							'''simple docstring'''




							def constraint_to_multiple_of(_lowerCamelCase				, _lowerCamelCase				, _lowerCamelCase=0				, _lowerCamelCase=None  ):
														_lowerCAmelCase     :							Tuple    =    round(val / multiple  ) * multiple

														if max_val is not None and x > max_val:
																					_lowerCAmelCase     :							Optional[int]    =    math.floor(val / multiple  ) * multiple

														if x < min_val:
																					_lowerCAmelCase     :							List[str]    =    math.ceil(val / multiple  ) * multiple

														return x

							_lowerCAmelCase     :							Union[str, Any]    =    (output_size, output_size) if isinstance(_lowerCamelCase				, _lowerCamelCase  ) else output_size

							_lowerCAmelCase   ,    _lowerCAmelCase     :							Optional[Any]    =    get_image_size(_lowerCamelCase  )
							_lowerCAmelCase   ,    _lowerCAmelCase     :							Any    =    output_size

							# determine new height and width
							_lowerCAmelCase     :							List[Any]    =    output_height / input_height
							_lowerCAmelCase     :							Any    =    output_width / input_width

							if keep_aspect_ratio:
														# scale as little as possible
														if abs(1 - scale_width  ) < abs(1 - scale_height  ):
																					# fit width
																					_lowerCAmelCase     :							Union[str, Any]    =    scale_width
														else:
																					# fit height
																					_lowerCAmelCase     :							Union[str, Any]    =    scale_height

							_lowerCAmelCase     :							List[str]    =    constraint_to_multiple_of(scale_height * input_height				, multiple=_lowerCamelCase  )
							_lowerCAmelCase     :							Dict    =    constraint_to_multiple_of(scale_width * input_width				, multiple=_lowerCamelCase  )

							return (new_height, new_width)



class   UpperCAmelCase_    (       a):
							lowerCamelCase__   			=		['pixel_values']







							def __init__(   self,			__a = True,			__a = None,			__a = PILImageResampling.BILINEAR,			__a = False,			__a = 1,			__a = True,			__a = 1 / 255,			__a = True,			__a = None,			__a = None,			**__a,			):

														'''simple docstring'''

														super().__init__(**__a)
														_lowerCAmelCase     :							Any    =    size if size is not None else {"height": 384, "width": 384}
														_lowerCAmelCase     :							Optional[int]    =    get_size_dict(__a)
														_lowerCAmelCase     :							Optional[Any]    =    do_resize
														_lowerCAmelCase     :							Dict    =    size
														_lowerCAmelCase     :							Any    =    keep_aspect_ratio
														_lowerCAmelCase     :							str    =    ensure_multiple_of
														_lowerCAmelCase     :							str    =    resample
														_lowerCAmelCase     :							Dict    =    do_rescale
														_lowerCAmelCase     :							Optional[int]    =    rescale_factor
														_lowerCAmelCase     :							Dict    =    do_normalize
														_lowerCAmelCase     :							Dict    =    image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
														_lowerCAmelCase     :							int    =    image_std if image_std is not None else IMAGENET_STANDARD_STD







							def    snake_case__      (   self,			__a,			__a,			__a = False,			__a = 1,			__a = PILImageResampling.BICUBIC,			__a = None,			**__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							List[Any]    =    get_size_dict(__a)
														if "height" not in size or "width" not in size:
																					raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}")
														_lowerCAmelCase     :							List[Any]    =    get_resize_output_image_size(
														    __a,			output_size=(size["height"], size["width"]),			keep_aspect_ratio=__a,			multiple=__a,			)
														return resize(__a,			size=__a,			resample=__a,			data_format=__a,			**__a)







							def    snake_case__      (   self,			__a,			__a,			__a = None,			**__a,			):

														'''simple docstring'''

														return rescale(__a,			scale=__a,			data_format=__a,			**__a)







							def    snake_case__      (   self,			__a,			__a,			__a,			__a = None,			**__a,			):

														'''simple docstring'''

														return normalize(__a,			mean=__a,			std=__a,			data_format=__a,			**__a)







							def    snake_case__      (   self,			__a,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = None,			__a = ChannelDimension.FIRST,			**__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    do_resize if do_resize is not None else self.do_resize
														_lowerCAmelCase     :							List[Any]    =    size if size is not None else self.size
														_lowerCAmelCase     :							str    =    get_size_dict(__a)
														_lowerCAmelCase     :							Dict    =    keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
														_lowerCAmelCase     :							Any    =    ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
														_lowerCAmelCase     :							int    =    resample if resample is not None else self.resample
														_lowerCAmelCase     :							Union[str, Any]    =    do_rescale if do_rescale is not None else self.do_rescale
														_lowerCAmelCase     :							Tuple    =    rescale_factor if rescale_factor is not None else self.rescale_factor
														_lowerCAmelCase     :							List[str]    =    do_normalize if do_normalize is not None else self.do_normalize
														_lowerCAmelCase     :							Dict    =    image_mean if image_mean is not None else self.image_mean
														_lowerCAmelCase     :							List[str]    =    image_std if image_std is not None else self.image_std

														_lowerCAmelCase     :							Optional[Any]    =    make_list_of_images(__a)

														if not valid_images(__a):
																					raise ValueError(
																					    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
																					    "torch.Tensor, tf.Tensor or jax.ndarray.")

														if do_resize and size is None or resample is None:
																					raise ValueError("Size and resample must be specified if do_resize is True.")

														if do_rescale and rescale_factor is None:
																					raise ValueError("Rescale factor must be specified if do_rescale is True.")

														if do_normalize and (image_mean is None or image_std is None):
																					raise ValueError("Image mean and std must be specified if do_normalize is True.")

														# All transformations expect numpy arrays.
														_lowerCAmelCase     :							List[Any]    =    [to_numpy_array(__a) for image in images]

														if do_resize:
																					_lowerCAmelCase     :							Any    =    [self.resize(image=__a,			size=__a,			resample=__a) for image in images]

														if do_rescale:
																					_lowerCAmelCase     :							List[str]    =    [self.rescale(image=__a,			scale=__a) for image in images]

														if do_normalize:
																					_lowerCAmelCase     :							Dict    =    [self.normalize(image=__a,			mean=__a,			std=__a) for image in images]

														_lowerCAmelCase     :							List[str]    =    [to_channel_dimension_format(__a,			__a) for image in images]

														_lowerCAmelCase     :							Optional[Any]    =    {"pixel_values": images}
														return BatchFeature(data=__a,			tensor_type=__a)







							def    snake_case__      (   self,			__a,			__a = None):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[Any]    =    outputs.logits

														# Resize logits and compute semantic segmentation maps
														if target_sizes is not None:
																					if len(__a) != len(__a):
																												raise ValueError(
																												    "Make sure that you pass in as many target sizes as the batch dimension of the logits")

																					if is_torch_tensor(__a):
																												_lowerCAmelCase     :							List[Any]    =    target_sizes.numpy()

																					_lowerCAmelCase     :							Dict    =    []

																					for idx in range(len(__a)):
																												_lowerCAmelCase     :							int    =    torch.nn.functional.interpolate(
																												    logits[idx].unsqueeze(dim=0),			size=target_sizes[idx],			mode="bilinear",			align_corners=__a)
																												_lowerCAmelCase     :							int    =    resized_logits[0].argmax(dim=0)
																												semantic_segmentation.append(__a)
														else:
																					_lowerCAmelCase     :							Dict    =    logits.argmax(dim=1)
																					_lowerCAmelCase     :							str    =    [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]

														return semantic_segmentation
