



'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class 				SCREAMING_SNAKE_CASE(							datasets.BeamBasedBuilder			):







	"""simple docstring"""






	def      A	(						self			:     Dict		)			->		Tuple:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''content''': datasets.Value('''string'''		)}		)		,			supervised_keys=__snake_case		,			)







	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     List[str]		,			__snake_case			:     str		)			->		Optional[Any]:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={'''examples''': get_test_dummy_examples()}		)]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     Optional[Any]		)			->		Optional[Any]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case		)

class 				SCREAMING_SNAKE_CASE(							datasets.BeamBasedBuilder			):







	"""simple docstring"""






	def      A	(						self			:     Tuple		)			->		Union[str, Any]:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)		,			supervised_keys=__snake_case		,			)







	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     Optional[int]		,			__snake_case			:     Optional[Any]		)			->		Tuple:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={'''examples''': get_test_nested_examples()}		)
								]







	def      A	(						self			:     Tuple		,			__snake_case			:     Tuple		,			__snake_case			:     str		)			->		str:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case		)


def  snake_case_     (					)      ->   Optional[Any]:
							return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']  )]


def  snake_case_     (					)      ->   Union[str, Any]:
							return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']  )]




class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	@require_beam
	def      A	(						self			:     Tuple		)			->		Dict:
								UpperCAmelCase :       Dict 							=		len(get_test_dummy_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       str 							=		DummyBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train.arrow"""		)		)		)
															self.assertDictEqual(builder.info.features		,			datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
															UpperCAmelCase :       str 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															self.assertDictEqual(dset['''train'''][0]		,			get_test_dummy_examples()[0][1]		)
															self.assertDictEqual(
															    dset['''train'''][expected_num_examples - 1]		,			get_test_dummy_examples()[expected_num_examples - 1][1]		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset







	@require_beam
	def      A	(						self			:     List[Any]		)			->		List[Any]:
								import apache_beam as beam

								UpperCAmelCase :       Tuple 							=		beam.io.parquetio.WriteToParquet

								UpperCAmelCase :       List[str] 							=		len(get_test_dummy_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       Any 							=		DummyBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															with patch('''apache_beam.io.parquetio.WriteToParquet'''		) as write_parquet_mock:
																						UpperCAmelCase :       str 							=		partial(__snake_case		,			num_shards=2		)
																						builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            __snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train-00000-of-00002.arrow"""		)		)		)
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            __snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train-00000-of-00002.arrow"""		)		)		)
															self.assertDictEqual(builder.info.features		,			datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
															UpperCAmelCase :       Any 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															# Order is not preserved when sharding, so we just check that all the elements are there
															self.assertListEqual(sorted(dset['''train''']['''content''']		)		,			sorted(['''foo''', '''bar''', '''foobar''']		)		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset







	@require_beam
	def      A	(						self			:     Optional[Any]		)			->		List[str]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       Any 							=		DummyBeamDataset(cache_dir=__snake_case		)
															self.assertRaises(datasets.builder.MissingBeamOptions		,			builder.download_and_prepare		)







	@require_beam
	def      A	(						self			:     Optional[Any]		)			->		Optional[Any]:
								UpperCAmelCase :       List[str] 							=		len(get_test_nested_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       List[str] 							=		NestedBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train.arrow"""		)		)		)
															self.assertDictEqual(
															    builder.info.features		,			datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)		)
															UpperCAmelCase :       List[Any] 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															self.assertDictEqual(dset['''train'''][0]		,			get_test_nested_examples()[0][1]		)
															self.assertDictEqual(
															    dset['''train'''][expected_num_examples - 1]		,			get_test_nested_examples()[expected_num_examples - 1][1]		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset










'''simple docstring'''






import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin

class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	lowerCamelCase__			 =       """MCTCTFeatureExtractor"""
	lowerCamelCase__			 =       """AutoTokenizer"""







	def __init__(						self			:     Dict		,			__snake_case			:     Optional[int]		,			__snake_case			:     List[str]		)			->		str:
								super().__init__(__snake_case		,			__snake_case		)
								UpperCAmelCase :       List[Any] 							=		self.feature_extractor
								UpperCAmelCase :       Union[str, Any] 							=		False







	def __call__(						self			:     Any		,			*__snake_case			:     List[str]		,			**__snake_case			:     Any		)			->		List[Any]:
								# For backward compatibility
								if self._in_target_context_manager:
															return self.current_processor(*__snake_case		,			**__snake_case		)

								if "raw_speech" in kwargs:
															warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''		)
															UpperCAmelCase :       int 							=		kwargs.pop('''raw_speech'''		)
								else:
															UpperCAmelCase :       Union[str, Any] 							=		kwargs.pop('''audio'''		,			__snake_case		)
								UpperCAmelCase :       Optional[Any] 							=		kwargs.pop('''sampling_rate'''		,			__snake_case		)
								UpperCAmelCase :       Dict 							=		kwargs.pop('''text'''		,			__snake_case		)
								if len(__snake_case		) > 0:
															UpperCAmelCase :       Any 							=		args[0]
															UpperCAmelCase :       Optional[int] 							=		args[1:]

								if audio is None and text is None:
															raise ValueError('''You need to specify either an `audio` or `text` input to process.'''		)

								if audio is not None:
															UpperCAmelCase :       List[str] 							=		self.feature_extractor(__snake_case		,			*__snake_case		,			sampling_rate=__snake_case		,			**__snake_case		)
								if text is not None:
															UpperCAmelCase :       int 							=		self.tokenizer(__snake_case		,			**__snake_case		)

								if text is None:
															return inputs
								elif audio is None:
															return encodings
								else:
															UpperCAmelCase :       str 							=		encodings['''input_ids''']
															return inputs







	def      A	(						self			:     List[Any]		,			*__snake_case			:     List[Any]		,			**__snake_case			:     List[Any]		)			->		str:
								return self.tokenizer.batch_decode(*__snake_case		,			**__snake_case		)







	def      A	(						self			:     List[Any]		,			*__snake_case			:     int		,			**__snake_case			:     Optional[int]		)			->		Any:
								# For backward compatibility
								if self._in_target_context_manager:
															return self.current_processor.pad(*__snake_case		,			**__snake_case		)

								UpperCAmelCase :       List[Any] 							=		kwargs.pop('''input_features'''		,			__snake_case		)
								UpperCAmelCase :       Optional[Any] 							=		kwargs.pop('''labels'''		,			__snake_case		)
								if len(__snake_case		) > 0:
															UpperCAmelCase :       List[str] 							=		args[0]
															UpperCAmelCase :       List[Any] 							=		args[1:]

								if input_features is not None:
															UpperCAmelCase :       Tuple 							=		self.feature_extractor.pad(__snake_case		,			*__snake_case		,			**__snake_case		)
								if labels is not None:
															UpperCAmelCase :       Optional[int] 							=		self.tokenizer.pad(__snake_case		,			**__snake_case		)

								if labels is None:
															return input_features
								elif input_features is None:
															return labels
								else:
															UpperCAmelCase :       List[str] 							=		labels['''input_ids''']
															return input_features







	def      A	(						self			:     Union[str, Any]		,			*__snake_case			:     Optional[Any]		,			**__snake_case			:     Optional[int]		)			->		Optional[Any]:
								return self.tokenizer.decode(*__snake_case		,			**__snake_case		)







	@contextmanager
	def      A	(						self			:     Any		)			->		Optional[int]:
								warnings.warn(
								    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
								    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
								    '''your audio inputs, or in a separate call.'''		)
								UpperCAmelCase :       Dict 							=		True
								UpperCAmelCase :       List[Any] 							=		self.tokenizer
								yield
								UpperCAmelCase :       Tuple 							=		self.feature_extractor
								UpperCAmelCase :       List[Any] 							=		False





