



'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class 				SCREAMING_SNAKE_CASE(							datasets.BeamBasedBuilder			):







	"""simple docstring"""






	def      A	(						self			:     Dict		)			->		Tuple:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''content''': datasets.Value('''string'''		)}		)		,			supervised_keys=__snake_case		,			)







	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     List[str]		,			__snake_case			:     str		)			->		Optional[Any]:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={'''examples''': get_test_dummy_examples()}		)]







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     Optional[Any]		,			__snake_case			:     Optional[Any]		)			->		Optional[Any]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case		)

class 				SCREAMING_SNAKE_CASE(							datasets.BeamBasedBuilder			):







	"""simple docstring"""






	def      A	(						self			:     Tuple		)			->		Union[str, Any]:
								return datasets.DatasetInfo(
								    features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)		,			supervised_keys=__snake_case		,			)







	def      A	(						self			:     Union[str, Any]		,			__snake_case			:     Optional[int]		,			__snake_case			:     Optional[Any]		)			->		Tuple:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN		,			gen_kwargs={'''examples''': get_test_nested_examples()}		)
								]







	def      A	(						self			:     Tuple		,			__snake_case			:     Tuple		,			__snake_case			:     str		)			->		str:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(__snake_case		)


def  snake_case_     (					)      ->   Optional[Any]:
							return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']  )]


def  snake_case_     (					)      ->   Union[str, Any]:
							return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']  )]




class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	@require_beam
	def      A	(						self			:     Tuple		)			->		Dict:
								UpperCAmelCase :       Dict 							=		len(get_test_dummy_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       str 							=		DummyBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train.arrow"""		)		)		)
															self.assertDictEqual(builder.info.features		,			datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
															UpperCAmelCase :       str 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															self.assertDictEqual(dset['''train'''][0]		,			get_test_dummy_examples()[0][1]		)
															self.assertDictEqual(
															    dset['''train'''][expected_num_examples - 1]		,			get_test_dummy_examples()[expected_num_examples - 1][1]		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset







	@require_beam
	def      A	(						self			:     List[Any]		)			->		List[Any]:
								import apache_beam as beam

								UpperCAmelCase :       Tuple 							=		beam.io.parquetio.WriteToParquet

								UpperCAmelCase :       List[str] 							=		len(get_test_dummy_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       Any 							=		DummyBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															with patch('''apache_beam.io.parquetio.WriteToParquet'''		) as write_parquet_mock:
																						UpperCAmelCase :       str 							=		partial(__snake_case		,			num_shards=2		)
																						builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            __snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train-00000-of-00002.arrow"""		)		)		)
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            __snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train-00000-of-00002.arrow"""		)		)		)
															self.assertDictEqual(builder.info.features		,			datasets.Features({'''content''': datasets.Value('''string'''		)}		)		)
															UpperCAmelCase :       Any 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															# Order is not preserved when sharding, so we just check that all the elements are there
															self.assertListEqual(sorted(dset['''train''']['''content''']		)		,			sorted(['''foo''', '''bar''', '''foobar''']		)		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset







	@require_beam
	def      A	(						self			:     Optional[Any]		)			->		List[str]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       Any 							=		DummyBeamDataset(cache_dir=__snake_case		)
															self.assertRaises(datasets.builder.MissingBeamOptions		,			builder.download_and_prepare		)







	@require_beam
	def      A	(						self			:     Optional[Any]		)			->		Optional[Any]:
								UpperCAmelCase :       List[str] 							=		len(get_test_nested_examples()		)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase :       List[str] 							=		NestedBeamDataset(cache_dir=__snake_case		,			beam_runner='''DirectRunner'''		)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			F"""{builder.name}-train.arrow"""		)		)		)
															self.assertDictEqual(
															    builder.info.features		,			datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''		)}		)}		)		)
															UpperCAmelCase :       List[Any] 							=		builder.as_dataset()
															self.assertEqual(dset['''train'''].num_rows		,			__snake_case		)
															self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,			__snake_case		)
															self.assertDictEqual(dset['''train'''][0]		,			get_test_nested_examples()[0][1]		)
															self.assertDictEqual(
															    dset['''train'''][expected_num_examples - 1]		,			get_test_nested_examples()[expected_num_examples - 1][1]		)
															self.assertTrue(
															    os.path.exists(os.path.join(__snake_case		,			builder.name		,			'''default'''		,			'''0.0.0'''		,			'''dataset_info.json'''		)		)		)
															del dset











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_ : Dict    		=       {"configuration_mbart": ["MBART_PRETRAINED_CONFIG_ARCHIVE_MAP", "MBartConfig", "MBartOnnxConfig"]}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : Tuple    		=       ["MBartTokenizer"]

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : str    		=       ["MBartTokenizerFast"]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : List[Any]    		=       [
        "MBART_PRETRAINED_MODEL_ARCHIVE_LIST",
        "MBartForCausalLM",
        "MBartForConditionalGeneration",
        "MBartForQuestionAnswering",
        "MBartForSequenceClassification",
        "MBartModel",
        "MBartPreTrainedModel",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : Any    		=       [
        "TFMBartForConditionalGeneration",
        "TFMBartModel",
        "TFMBartPreTrainedModel",
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    snake_case_ : List[str]    		=       [
        "FlaxMBartForConditionalGeneration",
        "FlaxMBartForQuestionAnswering",
        "FlaxMBartForSequenceClassification",
        "FlaxMBartModel",
        "FlaxMBartPreTrainedModel",
    ]


if TYPE_CHECKING:
    from .configuration_mbart import MBART_PRETRAINED_CONFIG_ARCHIVE_MAP, MBartConfig, MBartOnnxConfig

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mbart import MBartTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_mbart_fast import MBartTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mbart import (
            MBART_PRETRAINED_MODEL_ARCHIVE_LIST,
            MBartForCausalLM,
            MBartForConditionalGeneration,
            MBartForQuestionAnswering,
            MBartForSequenceClassification,
            MBartModel,
            MBartPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_mbart import TFMBartForConditionalGeneration, TFMBartModel, TFMBartPreTrainedModel

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_mbart import (
            FlaxMBartForConditionalGeneration,
            FlaxMBartForQuestionAnswering,
            FlaxMBartForSequenceClassification,
            FlaxMBartModel,
            FlaxMBartPreTrainedModel,
        )

else:
    import sys

    snake_case_ : Any    		=       _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






