




import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
UpperCamelCase__						=		'\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'

# TODO: Add description of the dataset here
# You can copy an official description
UpperCamelCase__						=		'\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'

# TODO: Add a link to an official homepage for the dataset here
UpperCamelCase__						=		''

# TODO: Add the licence for the dataset here if you can find it
UpperCamelCase__						=		''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
UpperCamelCase__						=		{
    'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
    'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
}


class 							__SCREAMING_SNAKE_CASE				(						datasets.GeneratorBasedBuilder    ):
					_lowerCamelCase						    =					datasets.Version("""1.1.0"""    )

					# This is an example of a dataset with multiple configurations.
					# If you don't want/need to define several sub-sets in your dataset,
					# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

					# If you need to make complex sub-parts in the datasets with configurable options
					# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
					# BUILDER_CONFIG_CLASS = MyBuilderConfig

					# You will be able to load one or the other configurations in the following list with
					# data = datasets.load_dataset('my_dataset', 'first_domain')
					# data = datasets.load_dataset('my_dataset', 'second_domain')
					_lowerCamelCase						    =					[
					    datasets.BuilderConfig(name="""first_domain"""  ,	version=lowerCamelCase  ,	description="""This part of my dataset covers a first domain"""    ),
					    datasets.BuilderConfig(name="""second_domain"""  ,	version=lowerCamelCase  ,	description="""This part of my dataset covers a second domain"""    ),
					]

					_lowerCamelCase						    =					"""first_domain"""  # It's not mandatory to have a default configuration. Just use one if it make sense.

					def       lowercase_				(							self   ):







						'''simple docstring'''




						if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
							lowerCamelCase_				:						Any				  =				datasets.Features(
							    {
							        'sentence': datasets.Value('string'   ),
							        'option1': datasets.Value('string'   ),
							        'answer': datasets.Value('string'   )
							        # These are the features of your dataset like images, labels ...
							    }   )
						else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
							lowerCamelCase_				:						List[str]				  =				datasets.Features(
							    {
							        'sentence': datasets.Value('string'   ),
							        'option2': datasets.Value('string'   ),
							        'second_domain_answer': datasets.Value('string'   )
							        # These are the features of your dataset like images, labels ...
							    }   )
						return datasets.DatasetInfo(
						    # This is the description that will appear on the datasets page.
						    description=_DESCRIPTION					,			features=__lowercase					,			homepage=_HOMEPAGE					,			license=_LICENSE					,			citation=_CITATION					,			)

					def       lowercase_				(							self					,			__lowercase   ):







						'''simple docstring'''




						lowerCamelCase_				:						str				  =				_URLS[self.config.name]
						lowerCamelCase_				:						int				  =				dl_manager.download_and_extract(__lowercase   )
						return [
						    datasets.SplitGenerator(
						        name=datasets.Split.TRAIN					,			gen_kwargs={
						            'filepath': os.path.join(__lowercase					,			'train.jsonl'   ),
						            'split': 'train',
						        }					,			),
						    datasets.SplitGenerator(
						        name=datasets.Split.VALIDATION					,			gen_kwargs={
						            'filepath': os.path.join(__lowercase					,			'dev.jsonl'   ),
						            'split': 'dev',
						        }					,			),
						    datasets.SplitGenerator(
						        name=datasets.Split.TEST					,			gen_kwargs={
						            'filepath': os.path.join(__lowercase					,			'test.jsonl'   ),
						            'split': 'test'
						        }					,			),
						]





					def       lowercase_				(							self					,			__lowercase					,			__lowercase   ):







						'''simple docstring'''




						with open(__lowercase					,			encoding='utf-8'   ) as f:
							for key, row in enumerate(__lowercase   ):
								lowerCamelCase_				:						Optional[int]				  =				json.loads(__lowercase   )
								if self.config.name == "first_domain":
									# Yields examples as (key, example) tuples
									yield key, {
									    "sentence": data["sentence"],
									    "option1": data["option1"],
									    "answer": "" if split == "test" else data["answer"],
									}
								else:
									yield key, {
									    "sentence": data["sentence"],
									    "option2": data["option2"],
									    "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
									}






import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__						=		logging.get_logger(__name__)

UpperCamelCase__						=		{}


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					"""fsmt_decoder"""

					def __init__(							self					,			__lowercase=0					,			__lowercase=0   ):







						'''simple docstring'''




						super().__init__()
						lowerCamelCase_				:						Tuple				  =				vocab_size
						lowerCamelCase_				:						Union[str, Any]				  =				bos_token_id


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					"""fsmt"""
					_lowerCamelCase						    =					{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}

					def __init__(							self					,			__lowercase=["en", "de"]					,			__lowercase=42_024					,			__lowercase=42_024					,			__lowercase="relu"					,			__lowercase=1_024					,			__lowercase=200					,			__lowercase=1_024					,			__lowercase=4_096					,			__lowercase=12					,			__lowercase=16					,			__lowercase=0.0					,			__lowercase=4_096					,			__lowercase=12					,			__lowercase=16					,			__lowercase=0.0					,			__lowercase=0.0					,			__lowercase=0.1					,			__lowercase=0.0					,			__lowercase=0.02					,			__lowercase=2					,			__lowercase=True					,			__lowercase=True					,			__lowercase=False					,			__lowercase=5					,			__lowercase=1.0					,			__lowercase=False					,			__lowercase=True					,			__lowercase=1					,			__lowercase=0					,			__lowercase=2					,			__lowercase=2					,			**__lowercase					,			):







						'''simple docstring'''




						lowerCamelCase_				:						Any				  =				langs
						lowerCamelCase_				:						Union[str, Any]				  =				src_vocab_size
						lowerCamelCase_				:						Tuple				  =				tgt_vocab_size
						lowerCamelCase_				:						List[Any]				  =				d_model  # encoder_embed_dim and decoder_embed_dim

						lowerCamelCase_				:						str				  =				encoder_ffn_dim
						lowerCamelCase_				:						str				  =				encoder_layers
						lowerCamelCase_				:						Dict				  =				encoder_attention_heads
						lowerCamelCase_				:						Any				  =				encoder_layerdrop
						lowerCamelCase_				:						Union[str, Any]				  =				decoder_layerdrop
						lowerCamelCase_				:						Optional[int]				  =				decoder_ffn_dim
						lowerCamelCase_				:						List[str]				  =				decoder_layers
						lowerCamelCase_				:						int				  =				decoder_attention_heads
						lowerCamelCase_				:						List[str]				  =				max_position_embeddings
						lowerCamelCase_				:						int				  =				init_std  # Normal(0, this parameter)
						lowerCamelCase_				:						Optional[Any]				  =				activation_function

						lowerCamelCase_				:						Union[str, Any]				  =				DecoderConfig(vocab_size=__lowercase					,			bos_token_id=__lowercase   )
						if "decoder" in common_kwargs:
							del common_kwargs["decoder"]

						lowerCamelCase_				:						int				  =				scale_embedding  # scale factor will be sqrt(d_model) if True

						# 3 Types of Dropout
						lowerCamelCase_				:						Optional[int]				  =				attention_dropout
						lowerCamelCase_				:						Union[str, Any]				  =				activation_dropout
						lowerCamelCase_				:						Dict				  =				dropout

						lowerCamelCase_				:						Optional[int]				  =				use_cache
						super().__init__(
						    pad_token_id=__lowercase					,			bos_token_id=__lowercase					,			eos_token_id=__lowercase					,			decoder_start_token_id=__lowercase					,			is_encoder_decoder=__lowercase					,			tie_word_embeddings=__lowercase					,			forced_eos_token_id=__lowercase					,			max_length=__lowercase					,			num_beams=__lowercase					,			length_penalty=__lowercase					,			early_stopping=__lowercase					,			**__lowercase					,			)





					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						List[Any]				  =				copy.deepcopy(self.__dict__   )
						lowerCamelCase_				:						Optional[int]				  =				self.decoder.to_dict()
						lowerCamelCase_				:						Dict				  =				self.__class__.model_type
						return output
