
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import OwlViTImageProcessor
class       A__  (		unittest.TestCase							):





							def __init__(					self    :  str	,					_UpperCAmelCase    :  Union[str, Any]	,					_UpperCAmelCase    :  List[Any]=7	,					_UpperCAmelCase    :  List[str]=3	,					_UpperCAmelCase    :  Any=18	,					_UpperCAmelCase    :  Optional[Any]=30	,					_UpperCAmelCase    :  Dict=4_00	,					_UpperCAmelCase    :  str=True	,					_UpperCAmelCase    :  List[str]=None	,					_UpperCAmelCase    :  List[str]=True	,					_UpperCAmelCase    :  Optional[int]=None	,					_UpperCAmelCase    :  Tuple=True	,					_UpperCAmelCase    :  List[Any]=[0.48_145_466, 0.4_578_275, 0.40_821_073]	,					_UpperCAmelCase    :  List[str]=[0.26_862_954, 0.26_130_258, 0.27_577_711]	,					_UpperCAmelCase    :  Optional[Any]=True	,					)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   parent
											__lowercase								=   batch_size
											__lowercase								=   num_channels
											__lowercase								=   image_size
											__lowercase								=   min_resolution
											__lowercase								=   max_resolution
											__lowercase								=   do_resize
											__lowercase								=   size if size is not None else {'height': 18, 'width': 18}
											__lowercase								=   do_center_crop
											__lowercase								=   crop_size if crop_size is not None else {'height': 18, 'width': 18}
											__lowercase								=   do_normalize
											__lowercase								=   image_mean
											__lowercase								=   image_std
											__lowercase								=   do_convert_rgb





							def      a__					(					self    :  Dict							)		->						Tuple:



											"""simple docstring"""


											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_center_crop": self.do_center_crop,
											    "crop_size": self.crop_size,
											    "do_normalize": self.do_normalize,
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											    "do_convert_rgb": self.do_convert_rgb,
											}






@require_torch
@require_vision
class       A__  (		lowerCAmelCase__       ,		unittest.TestCase							):
							lowerCAmelCase__						:		Union[str, Any]            =				OwlViTImageProcessor if is_vision_available() else None





							def      a__					(					self    :  List[Any]							)		->						Dict:



											"""simple docstring"""


											__lowercase								=   OwlViTImageProcessingTester(self							)





							@property
							def      a__					(					self    :  Dict							)		->						Optional[Any]:



											"""simple docstring"""


											return self.image_processor_tester.prepare_image_processor_dict()





							def      a__					(					self    :  List[Any]							)		->						List[str]:



											"""simple docstring"""


											__lowercase								=   self.image_processing_class(**self.image_processor_dict							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'do_resize'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'size'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'do_center_crop'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'center_crop'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'do_normalize'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'image_mean'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'image_std'							)							)
											self.assertTrue(hasattr(_UpperCAmelCase	,					'do_convert_rgb'							)							)





							def      a__					(					self    :  str							)		->						Any:



											"""simple docstring"""


											__lowercase								=   self.image_processing_class.from_dict(self.image_processor_dict							)
											self.assertEqual(image_processor.size	,					{'height': 18, 'width': 18}							)
											self.assertEqual(image_processor.crop_size	,					{'height': 18, 'width': 18}							)

											__lowercase								=   self.image_processing_class.from_dict(self.image_processor_dict	,					size=42	,					crop_size=84							)
											self.assertEqual(image_processor.size	,					{'height': 42, 'width': 42}							)
											self.assertEqual(image_processor.crop_size	,					{'height': 84, 'width': 84}							)





							def      a__					(					self    :  Optional[int]							)		->						Optional[Any]:



											"""simple docstring"""


											__lowercase								=   self.image_processing_class(**self.image_processor_dict							)
											# create random PIL images
											__lowercase								=   prepare_image_inputs(self.image_processor_tester	,					equal_resolution=_UpperCAmelCase							)

											for image in image_inputs:
															self.assertIsInstance(_UpperCAmelCase	,					Image.Image							)

											# Test not batched input
											__lowercase								=   image_processing(image_inputs[0]	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)

											# Test batched
											__lowercase								=   image_processing(_UpperCAmelCase	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)





							def      a__					(					self    :  Any							)		->						Any:



											"""simple docstring"""


											__lowercase								=   self.image_processing_class(**self.image_processor_dict							)
											# create random numpy tensors
											__lowercase								=   prepare_image_inputs(self.image_processor_tester	,					equal_resolution=_UpperCAmelCase	,					numpify=_UpperCAmelCase							)
											for image in image_inputs:
															self.assertIsInstance(_UpperCAmelCase	,					np.ndarray							)

											# Test not batched input
											__lowercase								=   image_processing(image_inputs[0]	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)

											# Test batched
											__lowercase								=   image_processing(_UpperCAmelCase	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)





							def      a__					(					self    :  List[Any]							)		->						Dict:



											"""simple docstring"""


											__lowercase								=   self.image_processing_class(**self.image_processor_dict							)
											# create random PyTorch tensors
											__lowercase								=   prepare_image_inputs(self.image_processor_tester	,					equal_resolution=_UpperCAmelCase	,					torchify=_UpperCAmelCase							)
											for image in image_inputs:
															self.assertIsInstance(_UpperCAmelCase	,					torch.Tensor							)

											# Test not batched input
											__lowercase								=   image_processing(image_inputs[0]	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)

											# Test batched
											__lowercase								=   image_processing(_UpperCAmelCase	,					return_tensors='pt'							).pixel_values
											self.assertEqual(
											    encoded_images.shape	,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size['height'],
											        self.image_processor_tester.crop_size['width'],
											    )	,					)







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


SCREAMING_SNAKE_CASE__				=							datasets.utils.logging.get_logger(__name__)
@dataclass
class       A__  (		datasets.BuilderConfig							):
							lowerCAmelCase__						:		Optional[datasets.Features]            =				None
							lowerCAmelCase__						:		str            =				"utf-8"
							lowerCAmelCase__						:		Optional[str]            =				None
							lowerCAmelCase__						:		int            =				10 << 20  # 10MB
							lowerCAmelCase__						:		bool            =				False
							lowerCAmelCase__						:		str            =				"line"






class       A__  (		datasets.ArrowBasedBuilder							):
							lowerCAmelCase__						:		Optional[int]            =				TextConfig





							def      a__					(					self    :  List[str]							)		->						Any:



											"""simple docstring"""


											return datasets.DatasetInfo(features=self.config.features							)





							def      a__					(					self    :  Tuple	,					_UpperCAmelCase    :  str							)		->						List[str]:



											"""simple docstring"""


											if not self.config.data_files:
															raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
											__lowercase								=   dl_manager.download_and_extract(self.config.data_files							)
											if isinstance(_UpperCAmelCase	,					(str, list, tuple)							):
															__lowercase								=   data_files
															if isinstance(_UpperCAmelCase	,					_UpperCAmelCase							):
																			__lowercase								=   [files]
															__lowercase								=   [dl_manager.iter_files(_UpperCAmelCase							) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,					gen_kwargs={'files': files}							)]
											__lowercase								=   []
											for split_name, files in data_files.items():
															if isinstance(_UpperCAmelCase	,					_UpperCAmelCase							):
																			__lowercase								=   [files]
															__lowercase								=   [dl_manager.iter_files(_UpperCAmelCase							) for file in files]
															splits.append(datasets.SplitGenerator(name=_UpperCAmelCase	,					gen_kwargs={'files': files}							)							)
											return splits





							def      a__					(					self    :  int	,					_UpperCAmelCase    :  pa.Table							)		->						pa.Table:



											"""simple docstring"""


											if self.config.features is not None:
															__lowercase								=   self.config.features.arrow_schema
															if all(not require_storage_cast(_UpperCAmelCase							) for feature in self.config.features.values()							):
																			# cheaper cast
																			__lowercase								=   pa_table.cast(_UpperCAmelCase							)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			__lowercase								=   table_cast(_UpperCAmelCase	,					_UpperCAmelCase							)
															return pa_table
											else:
															return pa_table.cast(pa.schema({'text': pa.string()}							)							)





							def      a__					(					self    :  Union[str, Any]	,					_UpperCAmelCase    :  Optional[Any]							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   list(self.config.features							) if self.config.features is not None else ['text']
											for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase							)							):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(_UpperCAmelCase	,					encoding=self.config.encoding	,					errors=self.config.errors							) as f:
																			if self.config.sample_by == "line":
																							__lowercase								=   0
																							while True:
																											__lowercase								=   f.read(self.config.chunksize							)
																											if not batch:
																															break
																											batch += f.readline()  # finish current line
																											# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																											__lowercase								=   StringIO(_UpperCAmelCase							).readlines()
																											if not self.config.keep_linebreaks:
																															__lowercase								=   [line.rstrip('\n'							) for line in batch]
																											__lowercase								=   pa.Table.from_arrays([pa.array(_UpperCAmelCase							)]	,					names=_UpperCAmelCase							)
																											# Uncomment for debugging (will print the Arrow table size and elements)
																											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																											batch_idx += 1
																			elif self.config.sample_by == "paragraph":
																							__lowercase								=   0
																							__lowercase								=   ''
																							while True:
																											__lowercase								=   f.read(self.config.chunksize							)
																											if not new_batch:
																															break
																											batch += new_batch
																											batch += f.readline()  # finish current line
																											__lowercase								=   batch.split('\n\n'							)
																											__lowercase								=   pa.Table.from_arrays(
																											    [pa.array([example for example in batch[:-1] if example]							)]	,					names=_UpperCAmelCase							)
																											# Uncomment for debugging (will print the Arrow table size and elements)
																											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																											batch_idx += 1
																											__lowercase								=   batch[-1]
																							if batch:
																											__lowercase								=   pa.Table.from_arrays([pa.array([batch]							)]	,					names=_UpperCAmelCase							)
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																			elif self.config.sample_by == "document":
																							__lowercase								=   f.read()
																							__lowercase								=   pa.Table.from_arrays([pa.array([text]							)]	,					names=_UpperCAmelCase							)
																							yield file_idx, self._cast_table(_UpperCAmelCase							)





