


"""simple docstring"""






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   _lowercase	,      unittest.TestCase):




							'''simple docstring'''





							__magic_name__					:     int				  =  PreTrainedTokenizerFast
							__magic_name__					:     Tuple				  =  False
							__magic_name__					:     str				  =  True
							__magic_name__					:     Tuple				  =  '''tokenizer_file'''




							def        UpperCAmelCase (						self)   ->	Dict:







								'''simple docstring'''




								snake_case__  :						Union[str, Any]		=      False  # because we don't have pretrained_vocab_files_map
								super().setUp()
								snake_case__  :						Any		=      True

								snake_case__  :						List[str]		=      ["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
								snake_case__  :						List[str]		=      "SaulLu/dummy-tokenizer-bytelevel-bpe"

								# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
								snake_case__  :						Tuple		=      [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

								snake_case__  :						Union[str, Any]		=      PreTrainedTokenizerFast.from_pretrained(model_paths[0])
								tokenizer.save_pretrained(self.tmpdirname)




							def        UpperCAmelCase (						self)   ->	Optional[int]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	Optional[int]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	str:







								'''simple docstring'''




								pass




							def        UpperCAmelCase (						self)   ->	List[Any]:







								'''simple docstring'''




								snake_case__  :						Optional[Any]		=      self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										try:
											snake_case__  :						List[str]		=      tempfile.mkdtemp()
											snake_case__  :						Optional[Any]		=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,					**lowerCamelCase__)

											tokenizer.save_pretrained(self.tmpdirname)
											super().test_training_new_tokenizer()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname)
											snake_case__  :						List[Any]		=      tmpdirname_orig




							def        UpperCAmelCase (						self)   ->	Dict:







								'''simple docstring'''




								snake_case__  :						Tuple		=      self.tmpdirname
								# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										try:
											snake_case__  :						Optional[int]		=      tempfile.mkdtemp()
											snake_case__  :						str		=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__	,					**lowerCamelCase__)

											tokenizer.save_pretrained(self.tmpdirname)
											super().test_training_new_tokenizer_with_special_tokens_change()
										finally:
											# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
											# is restored
											shutil.rmtree(self.tmpdirname)
											snake_case__  :						int		=      tmpdirname_orig







							def        UpperCAmelCase (						self)   ->	int:







								'''simple docstring'''




								snake_case__  :						str		=      self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name)

								snake_case__  :						Dict		=      ("a" for _ in range(1_000))
								snake_case__  :						Any		=      tokenizer.train_new_from_iterator(text_iterator=lowerCamelCase__	,					length=1_000	,					vocab_size=50)

								snake_case__  :						Optional[Any]		=      new_tokenizer.encode("a🤗")
								self.assertEqual(lowerCamelCase__	,					[64, 172, 253, 97, 245])


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   unittest.TestCase):




							'''simple docstring'''





							def        UpperCAmelCase (						self)   ->	List[str]:







								'''simple docstring'''




								snake_case__  :						List[Any]		=      AutoTokenizer.from_pretrained("bert-base-cased")
								snake_case__  :						Dict		=      json.loads(tokenizer._tokenizer.to_str())
								snake_case__  :						Any		=      len(lowerCamelCase__)

								with tempfile.TemporaryDirectory() as tmp_dir:
									# Hack to save this in the tokenizer_config.json
									snake_case__  :						List[Any]		=      ["tokenizer.4.0.0.json"]
									tokenizer.save_pretrained(lowerCamelCase__)
									json.dump(lowerCamelCase__	,					open(os.path.join(lowerCamelCase__	,					"tokenizer.4.0.0.json")	,					"w"))

									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
									snake_case__  :						Tuple		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
									self.assertEqual(len(lowerCamelCase__)	,					len(lowerCamelCase__) + 1)
									snake_case__  :						Union[str, Any]		=      json.loads(new_tokenizer._tokenizer.to_str())
									self.assertIn("huggingface"	,					json_tokenizer["model"]["vocab"])

									# Will need to be adjusted if we reach v42 and this test is still here.
									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
									shutil.move(os.path.join(lowerCamelCase__	,					"tokenizer.4.0.0.json")	,					os.path.join(lowerCamelCase__	,					"tokenizer.42.0.0.json"))
									snake_case__  :						Optional[int]		=      ["tokenizer.42.0.0.json"]
									tokenizer.save_pretrained(lowerCamelCase__)
									snake_case__  :						List[str]		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
									self.assertEqual(len(lowerCamelCase__)	,					len(lowerCamelCase__))
									snake_case__  :						Any		=      json.loads(new_tokenizer._tokenizer.to_str())
									self.assertNotIn("huggingface"	,					json_tokenizer["model"]["vocab"])







							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								snake_case__  :						List[Any]		=      "hf-internal-testing/test-two-tokenizers"

								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
								snake_case__  :						Union[str, Any]		=      AutoTokenizer.from_pretrained(lowerCamelCase__)
								self.assertEqual(len(lowerCamelCase__)	,					28_997)
								snake_case__  :						Optional[Any]		=      json.loads(tokenizer._tokenizer.to_str())
								self.assertIn("huggingface"	,					json_tokenizer["model"]["vocab"])

								# Testing an older version by monkey-patching the version in the module it's used.
								import transformers as old_transformers

								snake_case__  :						Any		=      "3.0.0"
								snake_case__  :						Optional[Any]		=      old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCamelCase__)
								self.assertEqual(len(lowerCamelCase__)	,					28_996)
								snake_case__  :						Any		=      json.loads(old_tokenizer._tokenizer.to_str())
								self.assertNotIn("huggingface"	,					json_tokenizer["model"]["vocab"])


@require_tokenizers
class 			SCREAMING_SNAKE_CASE_  (   unittest.TestCase):




							'''simple docstring'''





							def        UpperCAmelCase (						self)   ->	Union[str, Any]:







								'''simple docstring'''




								snake_case__  :						str		=      PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel")
								snake_case__  :						Any		=      "The Matrix is a 1999 science fiction action film."

								with concurrent.futures.ThreadPoolExecutor() as executor:
									snake_case__  :						Dict		=      [executor.submit(self.fetch	,					lowerCamelCase__	,					lowerCamelCase__) for i in range(10)]
									snake_case__  :						Dict		=      [future.result() for future in futures]
									self.assertEqual(lowerCamelCase__	,					[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10)])







							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__)   ->	Union[str, Any]:







								'''simple docstring'''




								return tokenizer.encode(lowerCamelCase__	,					truncation="longest_first"	,					padding="longest")








from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


lowerCAmelCase__							=			logging.get_logger(__name__)


if is_vision_available():
      import PIL






class     __magic_name__     (							_snake_case     ):
 UpperCAmelCase				     =    ["""pixel_values"""]


 def __init__(    self		:			str      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			PILImageResampling = PILImageResampling.BICUBIC      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Union[int, float] = 1 / 2_5_5      ,    lowerCAmelCase__		:			bool = True      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			bool = True      ,    **lowerCAmelCase__		:			str      ,    )     ->   None:
   super().__init__(**lowerCAmelCase__     )
   UpperCAmelCase          =	size if size is not None else {"shortest_edge": 2_2_4}
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )

   UpperCAmelCase          =	do_resize
   UpperCAmelCase          =	size
   UpperCAmelCase          =	resample
   UpperCAmelCase          =	do_center_crop
   UpperCAmelCase          =	crop_size
   UpperCAmelCase          =	do_rescale
   UpperCAmelCase          =	rescale_factor
   UpperCAmelCase          =	do_normalize
   UpperCAmelCase          =	image_mean if image_mean is not None else OPENAI_CLIP_MEAN
   UpperCAmelCase          =	image_std if image_std is not None else OPENAI_CLIP_STD
   UpperCAmelCase          =	do_convert_rgb


 def _UpperCamelCase						(    self		:			Optional[Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Dict[str, int]      ,    lowerCAmelCase__		:			PILImageResampling = PILImageResampling.BICUBIC      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			Optional[int]      ,    )     ->   np.ndarray:
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	get_resize_output_image_size(
       lowerCAmelCase__      ,    size=(size["height"], size["width"])      ,    default_to_square=lowerCAmelCase__     )
   return resize(lowerCAmelCase__      ,    size=lowerCAmelCase__      ,    resample=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			List[Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Dict[str, int]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[Any]      ,    )     ->   np.ndarray:
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )
   return center_crop(lowerCAmelCase__      ,    size=(size["height"], size["width"])      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Any      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Union[int, float]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[Any]      ,    )     ->   Optional[int]:
   return rescale(lowerCAmelCase__      ,    scale=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			Union[str, Any]      ,    lowerCAmelCase__		:			np.ndarray      ,    lowerCAmelCase__		:			Union[float, List[float]]      ,    lowerCAmelCase__		:			Union[float, List[float]]      ,    lowerCAmelCase__		:			Optional[Union[str, ChannelDimension]] = None      ,    **lowerCAmelCase__		:			List[str]      ,    )     ->   np.ndarray:
   return normalize(lowerCAmelCase__      ,    mean=lowerCAmelCase__      ,    std=lowerCAmelCase__      ,    data_format=lowerCAmelCase__      ,    **lowerCAmelCase__     )


 def _UpperCamelCase						(    self		:			List[str]      ,    lowerCAmelCase__		:			ImageInput      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Dict[str, int] = None      ,    lowerCAmelCase__		:			PILImageResampling = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			int = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			float = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			Optional[Union[float, List[float]]] = None      ,    lowerCAmelCase__		:			bool = None      ,    lowerCAmelCase__		:			Optional[Union[str, TensorType]] = None      ,    lowerCAmelCase__		:			Optional[ChannelDimension] = ChannelDimension.FIRST      ,    **lowerCAmelCase__		:			int      ,    )     ->   PIL.Image.Image:
   UpperCAmelCase          =	do_resize if do_resize is not None else self.do_resize
   UpperCAmelCase          =	size if size is not None else self.size
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__      ,    default_to_square=lowerCAmelCase__     )
   UpperCAmelCase          =	resample if resample is not None else self.resample
   UpperCAmelCase          =	do_center_crop if do_center_crop is not None else self.do_center_crop
   UpperCAmelCase          =	crop_size if crop_size is not None else self.crop_size
   UpperCAmelCase          =	get_size_dict(lowerCAmelCase__     )
   UpperCAmelCase          =	do_rescale if do_rescale is not None else self.do_rescale
   UpperCAmelCase          =	rescale_factor if rescale_factor is not None else self.rescale_factor
   UpperCAmelCase          =	do_normalize if do_normalize is not None else self.do_normalize
   UpperCAmelCase          =	image_mean if image_mean is not None else self.image_mean
   UpperCAmelCase          =	image_std if image_std is not None else self.image_std
   UpperCAmelCase          =	do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

   UpperCAmelCase          =	make_list_of_images(lowerCAmelCase__     )

   if not valid_images(lowerCAmelCase__     ):
     raise ValueError(
         "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
         "torch.Tensor, tf.Tensor or jax.ndarray."     )

   if do_resize and size is None:
     raise ValueError("Size must be specified if do_resize is True."     )

   if do_center_crop and crop_size is None:
     raise ValueError("Crop size must be specified if do_center_crop is True."     )

   if do_rescale and rescale_factor is None:
     raise ValueError("Rescale factor must be specified if do_rescale is True."     )

   if do_normalize and (image_mean is None or image_std is None):
     raise ValueError("Image mean and std must be specified if do_normalize is True."     )

   # PIL RGBA images are converted to RGB
   if do_convert_rgb:
     UpperCAmelCase          =	[convert_to_rgb(lowerCAmelCase__     ) for image in images]

   # All transformations expect numpy arrays.
   UpperCAmelCase          =	[to_numpy_array(lowerCAmelCase__     ) for image in images]

   if do_resize:
     UpperCAmelCase          =	[self.resize(image=lowerCAmelCase__      ,    size=lowerCAmelCase__      ,    resample=lowerCAmelCase__     ) for image in images]

   if do_center_crop:
     UpperCAmelCase          =	[self.center_crop(image=lowerCAmelCase__      ,    size=lowerCAmelCase__     ) for image in images]

   if do_rescale:
     UpperCAmelCase          =	[self.rescale(image=lowerCAmelCase__      ,    scale=lowerCAmelCase__     ) for image in images]

   if do_normalize:
     UpperCAmelCase          =	[self.normalize(image=lowerCAmelCase__      ,    mean=lowerCAmelCase__      ,    std=lowerCAmelCase__     ) for image in images]

   UpperCAmelCase          =	[to_channel_dimension_format(lowerCAmelCase__      ,    lowerCAmelCase__     ) for image in images]

   UpperCAmelCase          =	{"pixel_values": images}
   return BatchFeature(data=lowerCAmelCase__      ,    tensor_type=lowerCAmelCase__     )



