import json
import os
import re
import warnings
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  	=      logging.get_logger(__name__)

lowerCAmelCase__  	=      {
    "source_spm": "source.spm",
    "target_spm": "target.spm",
    "vocab": "vocab.json",
    "target_vocab_file": "target_vocab.json",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowerCAmelCase__  	=      {
    "source_spm": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm"
    },
    "target_spm": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm"
    },
    "vocab": {
        "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json"
    },
    "tokenizer_config_file": {
        "Helsinki-NLP/opus-mt-en-de": (
            "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json"
        )
    },
}

lowerCAmelCase__  	=      {"Helsinki-NLP/opus-mt-en-de": 5_1_2}
lowerCAmelCase__  	=      {}
class 	_a  (							lowerCamelCase_					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					VOCAB_FILES_NAMES
	__SCREAMING_SNAKE_CASE              =					PRETRAINED_VOCAB_FILES_MAP
	__SCREAMING_SNAKE_CASE              =					PRETRAINED_INIT_CONFIGURATION
	__SCREAMING_SNAKE_CASE              =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__SCREAMING_SNAKE_CASE              =					['input_ids', 'attention_mask']
	__SCREAMING_SNAKE_CASE              =					re.compile('>>.+<<'					)  # type: re.Pattern


	def __init__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_="<unk>"       ,  lowerCAmelCase_="</s>"       ,  lowerCAmelCase_="<pad>"       ,  lowerCAmelCase_=512       ,  lowerCAmelCase_ = None       ,  lowerCAmelCase_=False       ,  **lowerCAmelCase_       ,  ):
		_lowercase   ={} if sp_model_kwargs is None else sp_model_kwargs

		super().__init__(
		    # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
		    source_lang=lowerCAmelCase_       ,  target_lang=lowerCAmelCase_       ,  unk_token=lowerCAmelCase_       ,  eos_token=lowerCAmelCase_       ,  pad_token=lowerCAmelCase_       ,  model_max_length=lowerCAmelCase_       ,  sp_model_kwargs=self.sp_model_kwargs       ,  target_vocab_file=lowerCAmelCase_       ,  separate_vocabs=lowerCAmelCase_       ,  **lowerCAmelCase_       ,  )
		assert Path(lowerCAmelCase_       ).exists(), F'''cannot find spm source {source_spm}'''

		_lowercase   =separate_vocabs
		_lowercase   =load_json(lowerCAmelCase_       )
		if self.unk_token not in self.encoder:
			raise KeyError("<unk> token must be in vocab"       )
		assert self.pad_token in self.encoder

		if separate_vocabs:
			_lowercase   =load_json(lowerCAmelCase_       )
			_lowercase   ={v: k for k, v in self.target_encoder.items()}
			_lowercase   =[]
		else:
			_lowercase   ={v: k for k, v in self.encoder.items()}
			_lowercase   =[k for k in self.encoder if k.startswith(">>"       ) and k.endswith("<<"       )]

		_lowercase   =source_lang
		_lowercase   =target_lang
		_lowercase   =[source_spm, target_spm]

		# load SentencePiece model for pre-processing
		_lowercase   =load_spm(lowerCAmelCase_       ,  self.sp_model_kwargs       )
		_lowercase   =load_spm(lowerCAmelCase_       ,  self.sp_model_kwargs       )
		_lowercase   =self.spm_source
		_lowercase   =self.encoder

		# Multilingual target side: default to using first supported language code.

		self._setup_normalizer()


	def __lowerCAmelCase      (   self       ):
		try:
			from sacremoses import MosesPunctNormalizer

			_lowercase   =MosesPunctNormalizer(self.source_lang       ).normalize
		except (ImportError, FileNotFoundError):
			warnings.warn("Recommended: pip install sacremoses."       )
			_lowercase   =lambda lowerCAmelCase_       : x


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		return self.punc_normalizer(lowerCAmelCase_       ) if x else ""


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		return self.current_encoder.get(lowerCAmelCase_       ,  self.current_encoder[self.unk_token]       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =self.language_code_re.match(lowerCAmelCase_       )
		_lowercase   =[match.group(0       )] if match else []
		return code, self.language_code_re.sub(""       ,  lowerCAmelCase_       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   ,  _lowercase   =self.remove_language_code(lowerCAmelCase_       )
		_lowercase   =self.current_spm.encode(lowerCAmelCase_       ,  out_type=lowerCAmelCase_       )
		return code + pieces


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		return self.decoder.get(lowerCAmelCase_       ,  self.unk_token       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  **lowerCAmelCase_       ):
		return super().batch_decode(lowerCAmelCase_       ,  **lowerCAmelCase_       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  **lowerCAmelCase_       ):
		return super().decode(lowerCAmelCase_       ,  **lowerCAmelCase_       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =self.spm_source if self._decode_use_source_tokenizer else self.spm_target
		_lowercase   =[]
		_lowercase   =""
		for token in tokens:
			# make sure that special tokens are not decoded using sentencepiece model
			if token in self.all_special_tokens:
				out_string += sp_model.decode_pieces(lowerCAmelCase_       ) + token + " "
				_lowercase   =[]
			else:
				current_sub_tokens.append(lowerCAmelCase_       )
		out_string += sp_model.decode_pieces(lowerCAmelCase_       )
		return out_string.strip()


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=None       ):
		if token_ids_a is None:
			return token_ids_a + [self.eos_token_id]
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return token_ids_a + token_ids_a + [self.eos_token_id]


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.spm_source
		_lowercase   =self.encoder


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.spm_target
		if self.separate_vocabs:
			_lowercase   =self.target_encoder


	@property
	def __lowerCAmelCase      (   self       ):
		return len(self.encoder       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None       ):
		if not os.path.isdir(lowerCAmelCase_       ):
			logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
			return
		_lowercase   =[]

		if self.separate_vocabs:
			_lowercase   =os.path.join(
			    lowerCAmelCase_       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"]       ,  )
			_lowercase   =os.path.join(
			    lowerCAmelCase_       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["target_vocab_file"]       ,  )
			save_json(self.encoder       ,  lowerCAmelCase_       )
			save_json(self.target_encoder       ,  lowerCAmelCase_       )
			saved_files.append(lowerCAmelCase_       )
			saved_files.append(lowerCAmelCase_       )
		else:
			_lowercase   =os.path.join(
			    lowerCAmelCase_       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"]       )
			save_json(self.encoder       ,  lowerCAmelCase_       )
			saved_files.append(lowerCAmelCase_       )

		for spm_save_filename, spm_orig_path, spm_model in zip(
		    [VOCAB_FILES_NAMES["source_spm"], VOCAB_FILES_NAMES["target_spm"]]       ,  self.spm_files       ,  [self.spm_source, self.spm_target]       ,  ):
			_lowercase   =os.path.join(
			    lowerCAmelCase_       ,  (filename_prefix + "-" if filename_prefix else "") + spm_save_filename       )
			if os.path.abspath(lowerCAmelCase_       ) != os.path.abspath(lowerCAmelCase_       ) and os.path.isfile(lowerCAmelCase_       ):
				copyfile(lowerCAmelCase_       ,  lowerCAmelCase_       )
				saved_files.append(lowerCAmelCase_       )
			elif not os.path.isfile(lowerCAmelCase_       ):
				with open(lowerCAmelCase_       ,  "wb"       ) as fi:
					_lowercase   =spm_model.serialized_model_proto()
					fi.write(lowerCAmelCase_       )
				saved_files.append(lowerCAmelCase_       )

		return tuple(lowerCAmelCase_       )


	def __lowerCAmelCase      (   self       ):
		return self.get_src_vocab()


	def __lowerCAmelCase      (   self       ):
		return dict(self.encoder       ,  **self.added_tokens_encoder       )


	def __lowerCAmelCase      (   self       ):
		return dict(self.target_encoder       ,  **self.added_tokens_decoder       )


	def __getstate__(   self       ):
		_lowercase   =self.__dict__.copy()
		state.update(
		    {k: None for k in ["spm_source", "spm_target", "current_spm", "punc_normalizer", "target_vocab_file"]}       )
		return state


	def __setstate__(   self       ,  lowerCAmelCase_       ):
		_lowercase   =d

		# for backward compatibility
		if not hasattr(self       ,  "sp_model_kwargs"       ):
			_lowercase   ={}

		_lowercase   ,  _lowercase   =(load_spm(lowerCAmelCase_       ,  self.sp_model_kwargs       ) for f in self.spm_files)
		_lowercase   =self.spm_source
		self._setup_normalizer()


	def __lowerCAmelCase      (   self       ,  *lowerCAmelCase_       ,  **lowerCAmelCase_       ):
		return 1


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   =set(self.all_special_ids       )  # call it once instead of inside list comp
		all_special_ids.remove(self.unk_token_id       )  # <unk> is only sometimes special
		return [1 if x in all_special_ids else 0 for x in seq]


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_ = None       ,  lowerCAmelCase_ = False       ):
		if already_has_special_tokens:
			return self._special_token_mask(lowerCAmelCase_       )
		elif token_ids_a is None:
			return self._special_token_mask(lowerCAmelCase_       ) + [1]
		else:
			return self._special_token_mask(token_ids_a + token_ids_a       ) + [1]

def 	__lowerCamelCase							(				__a    : str      ,							__a    : Dict[str, Any]		)      ->       sentencepiece.SentencePieceProcessor:
	_lowercase   =sentencepiece.SentencePieceProcessor(**__a		)
	spm.Load(__a		)
	return spm

def 	__lowerCamelCase							(				__a    : str      ,							__a    : str		)      ->       None:
	with open(__a      ,							"w"		) as f:
		json.dump(__a      ,							__a      ,							indent=2		)

def 	__lowerCamelCase							(				__a    : str		)      ->       Union[Dict, List]:
	with open(__a      ,							"r"		) as f:
		return json.load(__a		)





"""simple docstring"""

from copy import deepcopy
from unittest.case import TestCase

import pytest

from datasets.arrow_dataset import Dataset
from datasets.features import Audio, ClassLabel, Features, Image, Sequence, Value
from datasets.info import DatasetInfo
from datasets.tasks import (
    AudioClassification,
    AutomaticSpeechRecognition,
    ImageClassification,
    LanguageModeling,
    QuestionAnsweringExtractive,
    Summarization,
    TextClassification,
    task_template_from_dict,
)
from datasets.utils.py_utils import asdict


__lowercase    =				{
    '''id''': '''5733be284776f41900661182''',
    '''title''': '''University_of_Notre_Dame''',
    '''context''': '''Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.''',
    '''question''': '''To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?''',
    '''answers''': {'''text''': ['''Saint Bernadette Soubirous'''], '''answer_start''': [515]},
}


@pytest.mark.parametrize(
    '''task_cls'''							,					[
        AudioClassification,
        AutomaticSpeechRecognition,
        ImageClassification,
        LanguageModeling,
        QuestionAnsweringExtractive,
        Summarization,
        TextClassification,
    ]							,					)
def    lowerCAmelCase			(__UpperCamelCase							:    List[str]			):
							"""simple docstring"""


							__UpperCamelCase							=task_cls()
							__UpperCamelCase							=asdict(__UpperCamelCase			)
							__UpperCamelCase							=task_template_from_dict(__UpperCamelCase			)
							assert task == reloaded






class 		_lowercase :



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=LanguageModeling(text_column='''input_text'''				)
													assert {"input_text": "text"} == task.column_mapping





						def   UpperCAmelCase_ (				self							:    str				)				->				int:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''text''': Value('''string'''				)}				)
													__UpperCamelCase							={'''text_column''': '''input_text'''}
													__UpperCamelCase							=LanguageModeling.from_dict(UpperCamelCase__				)
													assert "language-modeling" == task.task
													assert input_schema == task.input_schema






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Optional[Any]				)				->				Dict:
													'''simple docstring'''



													__UpperCamelCase							=sorted(['''pos''', '''neg''']				)





						def   UpperCAmelCase_ (				self							:    Tuple				)				->				Tuple:
													'''simple docstring'''



													__UpperCamelCase							=TextClassification(text_column='''input_text'''     , label_column='''input_label'''				)
													self.assertDictEqual({'''input_text''': '''text''', '''input_label''': '''labels'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				Union[str, Any]:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''text''': Value('''string'''				)}				)
													# Labels are cast to tuple during `TextClassification.__post_init__`, so we do the same here
													__UpperCamelCase							=Features({'''labels''': ClassLabel}				)
													__UpperCamelCase							={'''text_column''': '''input_text''', '''label_column''': '''input_labels'''}
													__UpperCamelCase							=TextClassification.from_dict(UpperCamelCase__				)
													self.assertEqual('''text-classification'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)





						def   UpperCAmelCase_ (				self							:    Tuple				)				->				Optional[Any]:
													'''simple docstring'''



													__UpperCamelCase							=TextClassification(text_column='''input_text'''     , label_column='''input_label'''				)
													self.assertEqual(task.label_schema['''labels''']     , UpperCamelCase__				)
													__UpperCamelCase							=task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels				)}				)				)
													self.assertEqual(task.label_schema['''labels''']     , ClassLabel(names=self.labels				)				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Dict				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=QuestionAnsweringExtractive(
													    context_column='''input_context'''     , question_column='''input_question'''     , answers_column='''input_answers'''				)
													self.assertDictEqual(
													    {'''input_context''': '''context''', '''input_question''': '''question''', '''input_answers''': '''answers'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    Optional[Any]				)				->				Optional[Any]:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''question''': Value('''string'''				), '''context''': Value('''string'''				)}				)
													__UpperCamelCase							=Features(
													    {
													        '''answers''': Sequence(
													            {
													                '''text''': Value('''string'''				),
													                '''answer_start''': Value('''int32'''				),
													            }				)
													    }				)
													__UpperCamelCase							={
													    '''context_column''': '''input_input_context''',
													    '''question_column''': '''input_question''',
													    '''answers_column''': '''input_answers''',
													}
													__UpperCamelCase							=QuestionAnsweringExtractive.from_dict(UpperCamelCase__				)
													self.assertEqual('''question-answering-extractive'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    List[Any]				)				->				str:
													'''simple docstring'''



													__UpperCamelCase							=Summarization(text_column='''input_text'''     , summary_column='''input_summary'''				)
													self.assertDictEqual({'''input_text''': '''text''', '''input_summary''': '''summary'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    int				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''text''': Value('''string'''				)}				)
													__UpperCamelCase							=Features({'''summary''': Value('''string'''				)}				)
													__UpperCamelCase							={'''text_column''': '''input_text''', '''summary_column''': '''input_summary'''}
													__UpperCamelCase							=Summarization.from_dict(UpperCamelCase__				)
													self.assertEqual('''summarization'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    List[Any]				)				->				Dict:
													'''simple docstring'''



													__UpperCamelCase							=AutomaticSpeechRecognition(audio_column='''input_audio'''     , transcription_column='''input_transcription'''				)
													self.assertDictEqual({'''input_audio''': '''audio''', '''input_transcription''': '''transcription'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				List[str]:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''audio''': Audio()}				)
													__UpperCamelCase							=Features({'''transcription''': Value('''string'''				)}				)
													__UpperCamelCase							={
													    '''audio_column''': '''input_audio''',
													    '''transcription_column''': '''input_transcription''',
													}
													__UpperCamelCase							=AutomaticSpeechRecognition.from_dict(UpperCamelCase__				)
													self.assertEqual('''automatic-speech-recognition'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    List[str]				)				->				List[str]:
													'''simple docstring'''



													__UpperCamelCase							=sorted(['''pos''', '''neg''']				)





						def   UpperCAmelCase_ (				self							:    Union[str, Any]				)				->				int:
													'''simple docstring'''



													__UpperCamelCase							=AudioClassification(audio_column='''input_audio'''     , label_column='''input_label'''				)
													self.assertDictEqual({'''input_audio''': '''audio''', '''input_label''': '''labels'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    Dict				)				->				Any:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''audio''': Audio()}				)
													__UpperCamelCase							=Features({'''labels''': ClassLabel}				)
													__UpperCamelCase							={
													    '''audio_column''': '''input_image''',
													    '''label_column''': '''input_label''',
													}
													__UpperCamelCase							=AudioClassification.from_dict(UpperCamelCase__				)
													self.assertEqual('''audio-classification'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)





						def   UpperCAmelCase_ (				self							:    Tuple				)				->				List[str]:
													'''simple docstring'''



													__UpperCamelCase							=AudioClassification(audio_column='''input_audio'''     , label_column='''input_label'''				)
													self.assertEqual(task.label_schema['''labels''']     , UpperCamelCase__				)
													__UpperCamelCase							=task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels				)}				)				)
													self.assertEqual(task.label_schema['''labels''']     , ClassLabel(names=self.labels				)				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Optional[Any]				)				->				str:
													'''simple docstring'''



													__UpperCamelCase							=sorted(['''pos''', '''neg''']				)





						def   UpperCAmelCase_ (				self							:    str				)				->				Tuple:
													'''simple docstring'''



													__UpperCamelCase							=ImageClassification(image_column='''input_image'''     , label_column='''input_label'''				)
													self.assertDictEqual({'''input_image''': '''image''', '''input_label''': '''labels'''}     , task.column_mapping				)





						def   UpperCAmelCase_ (				self							:    Optional[Any]				)				->				List[str]:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''image''': Image()}				)
													__UpperCamelCase							=Features({'''labels''': ClassLabel}				)
													__UpperCamelCase							={
													    '''image_column''': '''input_image''',
													    '''label_column''': '''input_label''',
													}
													__UpperCamelCase							=ImageClassification.from_dict(UpperCamelCase__				)
													self.assertEqual('''image-classification'''     , task.task				)
													self.assertEqual(UpperCamelCase__     , task.input_schema				)
													self.assertEqual(UpperCamelCase__     , task.label_schema				)





						def   UpperCAmelCase_ (				self							:    str				)				->				Optional[int]:
													'''simple docstring'''



													__UpperCamelCase							=ImageClassification(image_column='''input_image'''     , label_column='''input_label'''				)
													self.assertEqual(task.label_schema['''labels''']     , UpperCamelCase__				)
													__UpperCamelCase							=task.align_with_features(Features({'''input_label''': ClassLabel(names=self.labels				)}				)				)
													self.assertEqual(task.label_schema['''labels''']     , ClassLabel(names=self.labels				)				)






class 		_lowercase (						__a      ):



						"""simple docstring"""




						def   UpperCAmelCase_ (				self							:    Optional[Any]				)				->				List[Any]:
													'''simple docstring'''



													__UpperCamelCase							=DatasetInfo(task_templates=QuestionAnsweringExtractive()				)
													__UpperCamelCase							=Dataset.from_dict({k: [v] for k, v in SAMPLE_QUESTION_ANSWERING_EXTRACTIVE.items()}     , info=UpperCamelCase__				)
													assert isinstance(dataset.info.task_templates     , UpperCamelCase__				)
													assert len(dataset.info.task_templates				) == 1

													def keep_task(UpperCamelCase__							:    Optional[int]				):
																				return x

													def dont_keep_task(UpperCamelCase__							:    Optional[int]				):
																				__UpperCamelCase							=deepcopy(UpperCamelCase__				)
																				__UpperCamelCase							=0
																				return out

													__UpperCamelCase							=dataset.map(UpperCamelCase__				)
													assert mapped_dataset.info.task_templates == dataset.info.task_templates
													# reload from cache
													__UpperCamelCase							=dataset.map(UpperCamelCase__				)
													assert mapped_dataset.info.task_templates == dataset.info.task_templates

													__UpperCamelCase							=dataset.map(UpperCamelCase__				)
													assert mapped_dataset.info.task_templates == []
													# reload from cache
													__UpperCamelCase							=dataset.map(UpperCamelCase__				)
													assert mapped_dataset.info.task_templates == []





						def   UpperCAmelCase_ (				self							:    List[str]				)				->				Dict:
													'''simple docstring'''



													__UpperCamelCase							=Features({'''text''': Value('''string'''				), '''label''': ClassLabel(names=('''pos''', '''neg''')				)}				)
													__UpperCamelCase							=TextClassification(text_column='''text'''     , label_column='''label'''				)
													__UpperCamelCase							=DatasetInfo(features=UpperCamelCase__     , task_templates=UpperCamelCase__				)
													__UpperCamelCase							=Dataset.from_dict({'''text''': ['''A sentence.'''], '''label''': ['''pos''']}     , info=UpperCamelCase__				)

													def process(UpperCamelCase__							:    Any				):
																				return example

													__UpperCamelCase							=dataset.remove_columns('''label'''				)
													__UpperCamelCase							=modified_dataset.map(UpperCamelCase__				)
													assert mapped_dataset.info.task_templates == []



