






'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCamelCase_					=				logging.get_logger(__name__)

UpperCamelCase_					=				{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

UpperCamelCase_					=				{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

UpperCamelCase_					=				{
    """openai-gpt""": 512,
}






def  lowerCAmelCase__		(					a_  :     List[str]			)   -> Any:
       UpperCAmelCase__	:   Optional[int]												=	text.strip()
       if not text:
              return []
       UpperCAmelCase__	:   Union[str, Any]												=	text.split()
       return tokens




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      def __init__(					self  ,		_UpperCAmelCase=True  ,		_UpperCAmelCase=None  ,		_UpperCAmelCase=True  ,		_UpperCAmelCase=None  ,		_UpperCAmelCase=True  ,		):
             if never_split is None:
                    UpperCAmelCase__	:   str												=	[]
             UpperCAmelCase__	:   Union[str, Any]												=	do_lower_case
             UpperCAmelCase__	:   Tuple												=	set(_UpperCAmelCase						)
             UpperCAmelCase__	:   Tuple												=	tokenize_chinese_chars
             UpperCAmelCase__	:   Tuple												=	strip_accents
             UpperCAmelCase__	:   Dict												=	do_split_on_punc




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase=None						):
             UpperCAmelCase__	:   str												=	self.never_split.union(set(_UpperCAmelCase						)						) if never_split else self.never_split
             UpperCAmelCase__	:   Union[str, Any]												=	self._clean_text(_UpperCAmelCase						)

             # This was added on November 1st, 2018 for the multilingual and Chinese
             # models. This is also applied to the English models now, but it doesn't
             # matter since the English models were not trained on any Chinese data
             # and generally don't have any Chinese data in them (there are Chinese
             # characters in the vocabulary because Wikipedia does have some Chinese
             # words in the English Wikipedia.).
             if self.tokenize_chinese_chars:
                    UpperCAmelCase__	:   int												=	self._tokenize_chinese_chars(_UpperCAmelCase						)
             # prevents treating the same character with different unicode codepoints as different characters
             UpperCAmelCase__	:   List[str]												=	unicodedata.normalize('''NFC'''  ,		_UpperCAmelCase						)
             UpperCAmelCase__	:   Union[str, Any]												=	whitespace_tokenize(_UpperCAmelCase						)
             UpperCAmelCase__	:   Optional[int]												=	[]
             for token in orig_tokens:
                    if token not in never_split:
                           if self.do_lower_case:
                                  UpperCAmelCase__	:   int												=	token.lower()
                                  if self.strip_accents is not False:
                                         UpperCAmelCase__	:   Union[str, Any]												=	self._run_strip_accents(_UpperCAmelCase						)
                           elif self.strip_accents:
                                  UpperCAmelCase__	:   Tuple												=	self._run_strip_accents(_UpperCAmelCase						)
                    split_tokens.extend(self._run_split_on_punc(_UpperCAmelCase  ,		_UpperCAmelCase						)						)

             UpperCAmelCase__	:   Tuple												=	whitespace_tokenize(''' '''.join(_UpperCAmelCase						)						)
             return output_tokens




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   List[Any]												=	unicodedata.normalize('''NFD'''  ,		_UpperCAmelCase						)
             UpperCAmelCase__	:   str												=	[]
             for char in text:
                    UpperCAmelCase__	:   str												=	unicodedata.category(_UpperCAmelCase						)
                    if cat == "Mn":
                           continue
                    output.append(_UpperCAmelCase						)
             return "".join(_UpperCAmelCase						)




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase=None						):
             if not self.do_split_on_punc or (never_split is not None and text in never_split):
                    return [text]
             UpperCAmelCase__	:   Optional[Any]												=	list(_UpperCAmelCase						)
             UpperCAmelCase__	:   Union[str, Any]												=	0
             UpperCAmelCase__	:   str												=	True
             UpperCAmelCase__	:   str												=	[]
             while i < len(_UpperCAmelCase						):
                    UpperCAmelCase__	:   Union[str, Any]												=	chars[i]
                    if _is_punctuation(_UpperCAmelCase						):
                           output.append([char]						)
                           UpperCAmelCase__	:   Dict												=	True
                    else:
                           if start_new_word:
                                  output.append([]						)
                           UpperCAmelCase__	:   Optional[Any]												=	False
                           output[-1].append(_UpperCAmelCase						)
                    i += 1

             return ["".join(_UpperCAmelCase						) for x in output]




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   Tuple												=	[]
             for char in text:
                    UpperCAmelCase__	:   List[Any]												=	ord(_UpperCAmelCase						)
                    if self._is_chinese_char(_UpperCAmelCase						):
                           output.append(''' '''						)
                           output.append(_UpperCAmelCase						)
                           output.append(''' '''						)
                    else:
                           output.append(_UpperCAmelCase						)
             return "".join(_UpperCAmelCase						)




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             # This defines a "chinese character" as anything in the CJK Unicode block:
             #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
             #
             # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
             # despite its name. The modern Korean Hangul alphabet is a different block,
             # as is Japanese Hiragana and Katakana. Those alphabets are used to write
             # space-separated words, so they are not treated specially and handled
             # like the all of the other languages.
             if (
                 (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
                 or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
                 or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
                 or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
                 or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
                 or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
                 or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
                 or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
             ):  #
                    return True

             return False




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   Union[str, Any]												=	[]
             for char in text:
                    UpperCAmelCase__	:   Dict												=	ord(_UpperCAmelCase						)
                    if cp == 0 or cp == 0XF_F_F_D or _is_control(_UpperCAmelCase						):
                           continue
                    if _is_whitespace(_UpperCAmelCase						):
                           output.append(''' '''						)
                    else:
                           output.append(_UpperCAmelCase						)
             return "".join(_UpperCAmelCase						)






def  lowerCAmelCase__		(					a_  :     Dict			)   -> List[str]:
       UpperCAmelCase__	:   str												=	set()
       UpperCAmelCase__	:   Optional[Any]												=	word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)			)
              UpperCAmelCase__	:   int												=	char
       return pairs






def  lowerCAmelCase__		(					a_  :     int			)   -> List[str]:
       UpperCAmelCase__	:   Dict												=	text.replace('''—'''			,						'''-'''			)
       UpperCAmelCase__	:   Union[str, Any]												=	text.replace('''–'''			,						'''-'''			)
       UpperCAmelCase__	:   List[Any]												=	text.replace('''―'''			,						'''-'''			)
       UpperCAmelCase__	:   List[str]												=	text.replace('''…'''			,						'''...'''			)
       UpperCAmelCase__	:   List[Any]												=	text.replace('''´'''			,						'''\''''			)
       UpperCAmelCase__	:   Tuple												=	re.sub(r'''(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''			,						r''' \1 '''			,						a_			)
       UpperCAmelCase__	:   Union[str, Any]												=	re.sub(r'''\s*\n\s*'''			,						''' \n '''			,						a_			)
       UpperCAmelCase__	:   Tuple												=	re.sub(r'''[^\S\n]+'''			,						''' '''			,						a_			)
       return text.strip()




class 					__UpperCAmelCase (		UpperCamelCase__	):

      '''simple docstring'''





      SCREAMING_SNAKE_CASE				:    Tuple				       =					VOCAB_FILES_NAMES
      SCREAMING_SNAKE_CASE				:    int				       =					PRETRAINED_VOCAB_FILES_MAP
      SCREAMING_SNAKE_CASE				:    str				       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      SCREAMING_SNAKE_CASE				:    str				       =					["input_ids", "attention_mask"]




      def __init__(					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase  ,		_UpperCAmelCase="<unk>"  ,		**_UpperCAmelCase						):
             super().__init__(unk_token=_UpperCAmelCase  ,		**_UpperCAmelCase						)

             try:
                    import ftfy
                    from spacy.lang.en import English

                    UpperCAmelCase__	:   Optional[int]												=	English()
                    UpperCAmelCase__	:   int												=	_nlp.tokenizer
                    UpperCAmelCase__	:   Optional[int]												=	ftfy.fix_text
             except ImportError:
                    logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''						)
                    UpperCAmelCase__	:   str												=	BasicTokenizer(do_lower_case=_UpperCAmelCase						)
                    UpperCAmelCase__	:   List[Any]												=	None

             with open(_UpperCAmelCase  ,		encoding='''utf-8'''						) as vocab_handle:
                    UpperCAmelCase__	:   Any												=	json.load(_UpperCAmelCase						)
             UpperCAmelCase__	:   Dict												=	{v: k for k, v in self.encoder.items()}
             with open(_UpperCAmelCase  ,		encoding='''utf-8'''						) as merges_handle:
                    UpperCAmelCase__	:   Optional[Any]												=	merges_handle.read().split('''\n'''						)[1:-1]
             UpperCAmelCase__	:   Any												=	[tuple(merge.split()						) for merge in merges]
             UpperCAmelCase__	:   int												=	dict(zip(_UpperCAmelCase  ,		range(len(_UpperCAmelCase						)						)						)						)
             UpperCAmelCase__	:   List[Any]												=	{}




      @property
      def 							lowerCamelCase     (					self						):
             return True




      @property
      def 							lowerCamelCase     (					self						):
             return len(self.encoder						)




      def 							lowerCamelCase     (					self						):
             return dict(self.encoder  ,		**self.added_tokens_encoder						)




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   int												=	tuple(token[:-1]						) + (token[-1] + '''</w>''',)
             if token in self.cache:
                    return self.cache[token]
             UpperCAmelCase__	:   Optional[Any]												=	get_pairs(_UpperCAmelCase						)

             if not pairs:
                    return token + "</w>"

             while True:
                    UpperCAmelCase__	:   Optional[Any]												=	min(_UpperCAmelCase  ,		key=lambda _UpperCAmelCase						: self.bpe_ranks.get(_UpperCAmelCase  ,		float('''inf'''						)						)						)
                    if bigram not in self.bpe_ranks:
                           break
                    UpperCAmelCase__							,							UpperCAmelCase__	:   str												=	bigram
                    UpperCAmelCase__	:   List[Any]												=	[]
                    UpperCAmelCase__	:   int												=	0
                    while i < len(_UpperCAmelCase						):
                           try:
                                  UpperCAmelCase__	:   Union[str, Any]												=	word.index(_UpperCAmelCase  ,		_UpperCAmelCase						)
                           except ValueError:
                                  new_word.extend(word[i:]						)
                                  break
                           else:
                                  new_word.extend(word[i:j]						)
                                  UpperCAmelCase__	:   Dict												=	j

                           if word[i] == first and i < len(_UpperCAmelCase						) - 1 and word[i + 1] == second:
                                  new_word.append(first + second						)
                                  i += 2
                           else:
                                  new_word.append(word[i]						)
                                  i += 1
                    UpperCAmelCase__	:   Dict												=	tuple(_UpperCAmelCase						)
                    UpperCAmelCase__	:   Optional[Any]												=	new_word
                    if len(_UpperCAmelCase						) == 1:
                           break
                    else:
                           UpperCAmelCase__	:   Optional[int]												=	get_pairs(_UpperCAmelCase						)
             UpperCAmelCase__	:   Optional[int]												=	''' '''.join(_UpperCAmelCase						)
             if word == "\n  </w>":
                    UpperCAmelCase__	:   int												=	'''\n</w>'''
             UpperCAmelCase__	:   Any												=	word
             return word




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   List[Any]												=	[]
             if self.fix_text is None:
                    # Using BERT's BasicTokenizer
                    UpperCAmelCase__	:   Dict												=	self.nlp.tokenize(_UpperCAmelCase						)
                    for token in text:
                           split_tokens.extend(list(self.bpe(_UpperCAmelCase						).split(''' '''						)						)						)
             else:
                    # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                    UpperCAmelCase__	:   Union[str, Any]												=	self.nlp(text_standardize(self.fix_text(_UpperCAmelCase						)						)						)
                    for token in text:
                           split_tokens.extend(list(self.bpe(token.text.lower()						).split(''' '''						)						)						)
             return split_tokens




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             return self.encoder.get(_UpperCAmelCase  ,		self.encoder.get(self.unk_token						)						)




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             return self.decoder.get(_UpperCAmelCase  ,		self.unk_token						)




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   List[Any]												=	''''''.join(_UpperCAmelCase						).replace('''</w>'''  ,		''' '''						).strip()
             return out_string




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase = None						):
             if not os.path.isdir(_UpperCAmelCase						):
                    logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
                    return
             UpperCAmelCase__	:   int												=	os.path.join(
                 _UpperCAmelCase  ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
             UpperCAmelCase__	:   Optional[int]												=	os.path.join(
                 _UpperCAmelCase  ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

             with open(_UpperCAmelCase  ,		'''w'''  ,		encoding='''utf-8'''						) as f:
                    f.write(json.dumps(self.encoder  ,		indent=2  ,		sort_keys=_UpperCAmelCase  ,		ensure_ascii=_UpperCAmelCase						) + '''\n'''						)

             UpperCAmelCase__	:   Union[str, Any]												=	0
             with open(_UpperCAmelCase  ,		'''w'''  ,		encoding='''utf-8'''						) as writer:
                    writer.write('''#version: 0.2\n'''						)
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,		key=lambda _UpperCAmelCase						: kv[1]						):
                           if index != token_index:
                                  logger.warning(
                                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                      ''' Please check that the tokenizer is not corrupted!'''						)
                                  UpperCAmelCase__	:   Any												=	token_index
                           writer.write(''' '''.join(_UpperCAmelCase						) + '''\n'''						)
                           index += 1

             return vocab_file, merge_file





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile, rmtree
from typing import Generator

import datasets.config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.filelock import logger as fl_logger
from datasets.utils.info_utils import VerificationMode
from datasets.utils.logging import ERROR, get_logger


__A    :				Union[str, Any]    						=	get_logger(__name__)
def        lowerCamelCase_     (     SCREAMING_SNAKE_CASE ):
			'''simple docstring'''
			return TestCommand(
			    args.dataset       ,						args.name       ,						args.cache_dir       ,						args.data_dir       ,						args.all_configs       ,						args.save_info or args.save_infos       ,						args.ignore_verifications       ,						args.force_redownload       ,						args.clear_cache       ,						)




class       UpperCAmelCase_   ( A							):




							'''simple docstring'''



							a__    	=  False  # to tell pytest it's not a test class





							@staticmethod
							def  _UpperCAmelCase  (       a							:						ArgumentParser      )      ->       str:
										SCREAMING_SNAKE_CASE     =       parser.add_parser("""test"""							,  help="""Test dataset implementation."""      )
										test_parser.add_argument("""--name"""							,  type=a							,  default=a							,  help="""Dataset processing name"""      )
										test_parser.add_argument(
										    """--cache_dir"""							,  type=a							,  default=a							,  help="""Cache directory where the datasets are stored."""							,  )
										test_parser.add_argument(
										    """--data_dir"""							,  type=a							,  default=a							,  help="""Can be used to specify a manual directory to get the files from."""							,  )
										test_parser.add_argument("""--all_configs"""							,  action="""store_true"""							,  help="""Test all dataset configurations"""      )
										test_parser.add_argument(
										    """--save_info"""							,  action="""store_true"""							,  help="""Save the dataset infos in the dataset card (README.md)"""      )
										test_parser.add_argument(
										    """--ignore_verifications"""							,  action="""store_true"""							,  help="""Run the test without checksums and splits checks."""							,  )
										test_parser.add_argument("""--force_redownload"""							,  action="""store_true"""							,  help="""Force dataset redownload"""      )
										test_parser.add_argument(
										    """--clear_cache"""							,  action="""store_true"""							,  help="""Remove downloaded files and cached datasets after each config test"""							,  )
										# aliases
										test_parser.add_argument("""--save_infos"""							,  action="""store_true"""							,  help="""alias to save_info"""      )
										test_parser.add_argument("""dataset"""							,  type=a							,  help="""Name of the dataset to download"""      )
										test_parser.set_defaults(func=_test_command_factory      )





							def __init__(       self							:						List[Any]							,  a							:						str							,  a							:						str							,  a							:						str							,  a							:						str							,  a							:						bool							,  a							:						bool							,  a							:						bool							,  a							:						bool							,  a							:						bool							,  )      ->       List[Any]:
										SCREAMING_SNAKE_CASE     =       dataset
										SCREAMING_SNAKE_CASE     =       name
										SCREAMING_SNAKE_CASE     =       cache_dir
										SCREAMING_SNAKE_CASE     =       data_dir
										SCREAMING_SNAKE_CASE     =       all_configs
										SCREAMING_SNAKE_CASE     =       save_infos
										SCREAMING_SNAKE_CASE     =       ignore_verifications
										SCREAMING_SNAKE_CASE     =       force_redownload
										SCREAMING_SNAKE_CASE     =       clear_cache
										if clear_cache and not cache_dir:
													print(
													    """When --clear_cache is used, specifying a cache directory is mandatory.\n"""
													    """The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n"""
													    """Please provide a --cache_dir that will be used to test the dataset script."""      )
													exit(1      )
										if save_infos:
													SCREAMING_SNAKE_CASE     =       True







							def  _UpperCAmelCase  (       self							:						List[str]      )      ->       Optional[int]:
										fl_logger().setLevel(a      )
										if self._name is not None and self._all_configs:
													print("""Both parameters `config` and `all_configs` can't be used at once."""      )
													exit(1      )
										SCREAMING_SNAKE_CASE  ,	SCREAMING_SNAKE_CASE     =       self._dataset, self._name
										SCREAMING_SNAKE_CASE     =       dataset_module_factory(a      )
										SCREAMING_SNAKE_CASE     =       import_main_class(module.module_path      )
										SCREAMING_SNAKE_CASE     =       len(builder_cls.BUILDER_CONFIGS      ) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1

										def get_builders() -> Generator[DatasetBuilder, None, None]:
													if self._all_configs and builder_cls.BUILDER_CONFIGS:
																for i, config in enumerate(builder_cls.BUILDER_CONFIGS      ):
																			if "config_name" in module.builder_kwargs:
																						yield builder_cls(
																						    cache_dir=self._cache_dir							,  data_dir=self._data_dir							,  **module.builder_kwargs							,  )
																			else:
																						yield builder_cls(
																						    config_name=config.name							,  cache_dir=self._cache_dir							,  data_dir=self._data_dir							,  **module.builder_kwargs							,  )
													else:
																if "config_name" in module.builder_kwargs:
																			yield builder_cls(cache_dir=self._cache_dir							,  data_dir=self._data_dir							,  **module.builder_kwargs      )
																else:
																			yield builder_cls(
																			    config_name=a							,  cache_dir=self._cache_dir							,  data_dir=self._data_dir							,  **module.builder_kwargs							,  )

										for j, builder in enumerate(get_builders()      ):
													print(f"""Testing builder '{builder.config.name}' ({j + 1}/{n_builders})"""      )
													SCREAMING_SNAKE_CASE     =       os.path.exists(
													    os.path.join(builder.get_imported_module_dir()							,  datasets.config.DATASETDICT_INFOS_FILENAME      )      )  # record checksums only if we need to update a (deprecated) dataset_infos.json
													builder.download_and_prepare(
													    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
													    if not self._force_redownload
													    else DownloadMode.FORCE_REDOWNLOAD							,  verification_mode=VerificationMode.NO_CHECKS
													    if self._ignore_verifications
													    else VerificationMode.ALL_CHECKS							,  try_from_hf_gcs=a							,  )
													builder.as_dataset()
													if self._save_infos:
																builder._save_infos()

													# If save_infos=True, the dataset card (README.md) is created next to the loaded module file.
													# The dataset_infos are saved in the YAML part of the README.md

													# Let's move it to the original directory of the dataset script, to allow the user to
													# upload them on S3 at the same time afterwards.
													if self._save_infos:
																SCREAMING_SNAKE_CASE     =       os.path.join(builder_cls.get_imported_module_dir()							,  """README.md"""      )
																SCREAMING_SNAKE_CASE     =       Path(a      ).name + """.py"""
																SCREAMING_SNAKE_CASE     =       os.path.join(a							,  a      )
																if os.path.isfile(a      ):
																			SCREAMING_SNAKE_CASE     =       os.path.dirname(a      )
																elif os.path.isfile(a      ):
																			SCREAMING_SNAKE_CASE     =       path
																elif os.path.isdir(a      ):  # for local directories containing only data files
																			SCREAMING_SNAKE_CASE     =       path
																else:  # in case of a remote dataset
																			SCREAMING_SNAKE_CASE     =       None
																			print(f"""Dataset card saved at {dataset_readme_path}"""      )

																# Move dataset_info back to the user
																if dataset_dir is not None:
																			SCREAMING_SNAKE_CASE     =       os.path.join(a							,  """README.md"""      )
																			copyfile(a							,  a      )
																			print(f"""Dataset card saved at {user_dataset_readme_path}"""      )

            # If clear_cache=True, the download folder and the dataset builder cache directory are deleted
													if self._clear_cache:
																if os.path.isdir(builder._cache_dir      ):
																			logger.warning(f"""Clearing cache at {builder._cache_dir}"""      )
																			rmtree(builder._cache_dir      )
																SCREAMING_SNAKE_CASE     =       os.path.join(self._cache_dir							,  datasets.config.DOWNLOADED_DATASETS_DIR      )
																if os.path.isdir(a      ):
																			logger.warning(f"""Clearing cache at {download_dir}"""      )
																			rmtree(a      )

										print("""Test successful."""      )






