






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase_     =			datasets.utils.logging.get_logger(__name__)
@dataclass
class   UpperCamelCase			(					datasets.BuilderConfig			):







		"""simple docstring"""



		snake_case			 = None
		snake_case			 = "utf-8"
		snake_case			 = None
		snake_case			 = 1_0 << 2_0  # 10MB
		snake_case			 = False
		snake_case			 = "line"


class   UpperCamelCase			(					datasets.ArrowBasedBuilder			):







		"""simple docstring"""



		snake_case			 = TextConfig
		def    A(	self		:							Any		)     -> Optional[int]:


						'''simple docstring'''
						return datasets.DatasetInfo(features=self.config.features		)
		def    A(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]		)     -> Union[str, Any]:


						'''simple docstring'''
						if not self.config.data_files:
										raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'		)
						A									=   dl_manager.download_and_extract(self.config.data_files		)
						if isinstance(a__    ,(str, list, tuple)		):
										A									=   data_files
										if isinstance(a__    ,a__		):
														A									=   [files]
										A									=   [dl_manager.iter_files(a__		) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,gen_kwargs={'files': files}		)]
						A									=   []
						for split_name, files in data_files.items():
										if isinstance(a__    ,a__		):
														A									=   [files]
										A									=   [dl_manager.iter_files(a__		) for file in files]
										splits.append(datasets.SplitGenerator(name=a__    ,gen_kwargs={'files': files}		)		)
						return splits
		def    A(	self		:							Optional[Any]    ,_SCREAMING_SNAKE_CASE		:							List[Any]		)     -> Tuple:


						'''simple docstring'''
						if self.config.features is not None:
										A									=   self.config.features.arrow_schema
										if all(not require_storage_cast(a__		) for feature in self.config.features.values()		):
														# cheaper cast
														A									=   pa_table.cast(a__		)
										else:
														# more expensive cast; allows str <-> int/float or str to Audio for example
														A									=   table_cast(a__    ,a__		)
										return pa_table
						else:
										return pa_table.cast(pa.schema({'text': pa.string()}		)		)







		def    A(	self		:							Optional[Any]    ,_SCREAMING_SNAKE_CASE		:							str		)     -> Union[str, Any]:


						'''simple docstring'''
						A									=   list(self.config.features		) if self.config.features is not None else ['text']
						for file_idx, file in enumerate(itertools.chain.from_iterable(a__		)		):
										# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
										with open(a__    ,encoding=self.config.encoding    ,errors=self.config.errors		) as f:
														if self.config.sample_by == "line":
																		A									=   0
																		while True:
																						A									=   f.read(self.config.chunksize		)
																						if not batch:
																										break
																						batch += f.readline()  # finish current line
																						# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																						A									=   StringIO(a__		).readlines()
																						if not self.config.keep_linebreaks:
																										A									=   [line.rstrip('\n'		) for line in batch]
																						A									=   pa.Table.from_arrays([pa.array(a__		)]    ,names=a__		)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(a__		)
																						batch_idx += 1
														elif self.config.sample_by == "paragraph":
																		A									=   0
																		A									=   ''
																		while True:
																						A									=   f.read(self.config.chunksize		)
																						if not new_batch:
																										break
																						batch += new_batch
																						batch += f.readline()  # finish current line
																						A									=   batch.split('\n\n'		)
																						A									=   pa.Table.from_arrays(
																						    [pa.array([example for example in batch[:-1] if example]		)]    ,names=a__		)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(a__		)
																						batch_idx += 1
																						A									=   batch[-1]
																		if batch:
																						A									=   pa.Table.from_arrays([pa.array([batch]		)]    ,names=a__		)
																						yield (file_idx, batch_idx), self._cast_table(a__		)
														elif self.config.sample_by == "document":
																		A									=   f.read()
																		A									=   pa.Table.from_arrays([pa.array([text]		)]    ,names=a__		)
																		yield file_idx, self._cast_table(a__		)












"""simple docstring"""






import copy
import inspect

from transformers.testing_utils import require_torch, torch_device
from transformers.utils.backbone_utils import BackboneType



@require_torch
class 						lowerCAmelCase       :
       lowerCAmelCase__    			=	()
       lowerCAmelCase__    			=	True






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.config_class

              # test default config
              _UpperCAmelCase			     =  config_class()
              self.assertIsNotNone(a__  )
              _UpperCAmelCase			     =  ['stem'] + [f"""stage{idx}""" for idx in range(1     ,	len(config.depths  ) + 1  )]
              self.assertEqual(config.stage_names     ,	a__  )
              self.assertTrue(set(config.out_features  ).issubset(set(config.stage_names  )  )  )

              # Test out_features and out_indices are correctly set
              # out_features and out_indices both None
              _UpperCAmelCase			     =  config_class(out_features=a__     ,	out_indices=a__  )
              self.assertEqual(config.out_features     ,	[config.stage_names[-1]]  )
              self.assertEqual(config.out_indices     ,	[len(config.stage_names  ) - 1]  )

              # out_features and out_indices both set
              _UpperCAmelCase			     =  config_class(out_features=['stem', 'stage1']     ,	out_indices=[0, 1]  )
              self.assertEqual(config.out_features     ,	['stem', 'stage1']  )
              self.assertEqual(config.out_indices     ,	[0, 1]  )

              # Only out_features set
              _UpperCAmelCase			     =  config_class(out_features=['stage1', 'stage3']  )
              self.assertEqual(config.out_features     ,	['stage1', 'stage3']  )
              self.assertEqual(config.out_indices     ,	[1, 3]  )

              # Only out_indices set
              _UpperCAmelCase			     =  config_class(out_indices=[0, 2]  )
              self.assertEqual(config.out_features     ,	[config.stage_names[0], config.stage_names[2]]  )
              self.assertEqual(config.out_indices     ,	[0, 2]  )

              # Error raised when out_indices do not correspond to out_features
              with self.assertRaises(a__  ):
                     _UpperCAmelCase			     =  config_class(out_features=['stage1', 'stage2']     ,	out_indices=[0, 2]  )






       def 				__A			( self  ):
              _UpperCAmelCase     ,				_UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     _UpperCAmelCase			     =  model_class(a__  )
                     _UpperCAmelCase			     =  inspect.signature(model.forward  )
                     # signature.parameters is an OrderedDict => so arg_names order is deterministic
                     _UpperCAmelCase			     =  [*signature.parameters.keys()]
                     _UpperCAmelCase			     =  ['pixel_values']
                     self.assertListEqual(arg_names[:1]     ,	a__  )






       def 				__A			( self  ):
              _UpperCAmelCase     ,				_UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     _UpperCAmelCase			     =  model_class(a__  )
                     self.assertEqual(len(model.channels  )     ,	len(config.out_features  )  )
                     _UpperCAmelCase			     =  model.num_features
                     _UpperCAmelCase			     =  [config.stage_names.index(a__  ) for feat in config.out_features]
                     _UpperCAmelCase			     =  [num_features[idx] for idx in out_indices]
                     self.assertListEqual(model.channels     ,	a__  )

                     _UpperCAmelCase			     =  copy.deepcopy(a__  )
                     _UpperCAmelCase			     =  None
                     _UpperCAmelCase			     =  model_class(a__  )
                     self.assertEqual(len(model.channels  )     ,	1  )
                     self.assertListEqual(model.channels     ,	[num_features[-1]]  )

                     _UpperCAmelCase			     =  copy.deepcopy(a__  )
                     _UpperCAmelCase			     =  None
                     _UpperCAmelCase			     =  model_class(a__  )
                     self.assertEqual(len(model.channels  )     ,	1  )
                     self.assertListEqual(model.channels     ,	[num_features[-1]]  )






       def 				__A			( self  ):
              _UpperCAmelCase     ,				_UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     _UpperCAmelCase			     =  model_class(a__  )
                     model.to(a__  )
                     model.eval()
                     _UpperCAmelCase			     =  model(**a__  )

                     self.assertEqual(len(result.feature_maps  )     ,	len(config.out_features  )  )
                     self.assertEqual(len(model.channels  )     ,	len(config.out_features  )  )
                     self.assertEqual(len(result.feature_maps  )     ,	len(config.out_indices  )  )
                     self.assertEqual(len(model.channels  )     ,	len(config.out_indices  )  )

                     # Check output of last stage is taken if out_features=None, out_indices=None
                     _UpperCAmelCase			     =  copy.deepcopy(a__  )
                     _UpperCAmelCase			     =  None
                     _UpperCAmelCase			     =  model_class(a__  )
                     model.to(a__  )
                     model.eval()
                     _UpperCAmelCase			     =  model(**a__  )

                     self.assertEqual(len(result.feature_maps  )     ,	1  )
                     self.assertEqual(len(model.channels  )     ,	1  )

                     _UpperCAmelCase			     =  copy.deepcopy(a__  )
                     _UpperCAmelCase			     =  None
                     _UpperCAmelCase			     =  model_class(a__  )
                     model.to(a__  )
                     model.eval()
                     _UpperCAmelCase			     =  model(**a__  )

                     self.assertEqual(len(result.feature_maps  )     ,	1  )
                     self.assertEqual(len(model.channels  )     ,	1  )

                     # Check backbone can be initialized with fresh weights
                     _UpperCAmelCase			     =  copy.deepcopy(a__  )
                     _UpperCAmelCase			     =  False
                     _UpperCAmelCase			     =  model_class(a__  )
                     model.to(a__  )
                     model.eval()
                     _UpperCAmelCase			     =  model(**a__  )






       def 				__A			( self  ):
              _UpperCAmelCase     ,				_UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs_for_common()

              for backbone_class in self.all_model_classes:
                     _UpperCAmelCase			     =  backbone_class(a__  )

                     self.assertTrue(hasattr(a__     ,	'backbone_type'  )  )
                     self.assertTrue(hasattr(a__     ,	'stage_names'  )  )
                     self.assertTrue(hasattr(a__     ,	'num_features'  )  )
                     self.assertTrue(hasattr(a__     ,	'out_indices'  )  )
                     self.assertTrue(hasattr(a__     ,	'out_features'  )  )
                     self.assertTrue(hasattr(a__     ,	'out_feature_channels'  )  )
                     self.assertTrue(hasattr(a__     ,	'channels'  )  )

                     self.assertIsInstance(backbone.backbone_type     ,	a__  )
                     # Verify num_features has been initialized in the backbone init
                     self.assertIsNotNone(backbone.num_features  )
                     self.assertTrue(len(backbone.channels  ) == len(backbone.out_indices  )  )
                     self.assertTrue(len(backbone.stage_names  ) == len(backbone.num_features  )  )
                     self.assertTrue(len(backbone.channels  ) <= len(backbone.num_features  )  )
                     self.assertTrue(len(backbone.out_feature_channels  ) == len(backbone.stage_names  )  )






       def 				__A			( self  ):
              _UpperCAmelCase     ,				_UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs_for_common()
              _UpperCAmelCase			     =  inputs_dict['pixel_values'].shape[0]

              for backbone_class in self.all_model_classes:
                     _UpperCAmelCase			     =  backbone_class(a__  )
                     backbone.to(a__  )
                     backbone.eval()

                     _UpperCAmelCase			     =  backbone(**a__  )

                     # Test default outputs and verify feature maps
                     self.assertIsInstance(outputs.feature_maps     ,	a__  )
                     self.assertTrue(len(outputs.feature_maps  ) == len(backbone.channels  )  )
                     for feature_map, n_channels in zip(outputs.feature_maps     ,	backbone.channels  ):
                            self.assertTrue(feature_map.shape[:2]     ,	(batch_size, n_channels)  )
                     self.assertIsNone(outputs.hidden_states  )
                     self.assertIsNone(outputs.attentions  )

                     # Test output_hidden_states=True
                     _UpperCAmelCase			     =  backbone(**a__     ,	output_hidden_states=a__  )
                     self.assertIsNotNone(outputs.hidden_states  )
                     self.assertTrue(len(outputs.hidden_states  )     ,	len(backbone.stage_names  )  )
                     for hidden_state, n_channels in zip(outputs.hidden_states     ,	backbone.channels  ):
                            self.assertTrue(hidden_state.shape[:2]     ,	(batch_size, n_channels)  )

                     # Test output_attentions=True
                     if self.has_attentions:
                            _UpperCAmelCase			     =  backbone(**a__     ,	output_attentions=a__  )
                            self.assertIsNotNone(outputs.attentions  )






