





'''simple docstring'''
import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


a__							:							Optional[int] =[
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.de'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.en'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.fr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.frr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.it'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.simple'''},
    {'''dataset''': '''snli''', '''config_name''': '''plain_text'''},
    {'''dataset''': '''eli5''', '''config_name''': '''LFQA_reddit'''},
    {'''dataset''': '''wiki40b''', '''config_name''': '''en'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.compressed'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.no_index'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.multiset.no_index'''},
    {'''dataset''': '''natural_questions''', '''config_name''': '''default'''},
]
def 				lowercase__   (   __lowercase						:  Optional[int]=True     ) ->      List[Any]:



       """simple docstring"""






       if with_config:
              return [
                  {
                      "testcase_name": d["dataset"] + "/" + d["config_name"],
                      "dataset": d["dataset"],
                      "config_name": d["config_name"],
                  }
                  for d in DATASETS_ON_HF_GCP
              ]
       else:
              return [
                  {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
              ]

@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=__lowerCamelCase       )       )
class      snake_case     (    __lowerCamelCase       ):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_	:					List[Any]					=None
 SCREAMING_SNAKE_CASE_	:					Tuple					=None






 def        _lowerCamelCase					(       self       :							int							,	__A       :							Dict							,	__A       :							List[Any]	):
        with TemporaryDirectory() as tmp_dir:
               __UpperCamelCase       =  dataset_module_factory(__A							,	cache_dir=__A	)

               __UpperCamelCase       =  import_main_class(dataset_module.module_path							,	dataset=__A	)

               __UpperCamelCase       =  builder_cls(
                   cache_dir=__A							,	config_name=__A							,	hash=dataset_module.hash							,	)

               __UpperCamelCase       =  '/'.join(
                   [
                       HF_GCP_BASE_URL,
                       builder_instance._relative_data_dir(with_hash=__A	).replace(os.sep							,	'/'	),
                       config.DATASET_INFO_FILENAME,
                   ]	)
               __UpperCamelCase       =  cached_path(__A							,	cache_dir=__A	)
               self.assertTrue(os.path.exists(__A	)	)






@pytest.mark.integration
def 				lowercase__   (   __lowercase						:  Union[str, Any]     ) ->      Optional[Any]:



       """simple docstring"""






       __UpperCamelCase       =  tmp_path_factory.mktemp('test_hf_gcp'     ) / 'test_wikipedia_simple'
       __UpperCamelCase       =  dataset_module_factory('wikipedia'				,  cache_dir=__lowercase     )
       __UpperCamelCase       =  import_main_class(dataset_module.module_path     )
       __UpperCamelCase       =  builder_cls(
           cache_dir=__lowercase				,  config_name='20220301.frr'				,  hash=dataset_module.hash				,  )
       # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
       __UpperCamelCase       =  None
       builder_instance.download_and_prepare()
       __UpperCamelCase       =  builder_instance.as_dataset()
       assert ds


@pytest.mark.integration
def 				lowercase__   (   __lowercase						:  Dict     ) ->      Any:



       """simple docstring"""






       __UpperCamelCase       =  dataset_module_factory('wikipedia'				,  cache_dir=__lowercase     )
       __UpperCamelCase       =  import_main_class(dataset_module.module_path				,  dataset=__lowercase     )
       __UpperCamelCase       =  builder_cls(
           cache_dir=__lowercase				,  config_name='20220301.frr'				,  hash=dataset_module.hash				,  )
       __UpperCamelCase       =  builder_instance.as_streaming_dataset()
       assert ds
       assert isinstance(__lowercase				,  __lowercase     )
       assert "train" in ds
       assert isinstance(ds['train']				,  __lowercase     )
       assert next(iter(ds['train']     )     )






from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_ : Any    		=       logging.get_logger(__name__)

snake_case_ : Optional[Any]    		=       {
    "tiiuae/falcon-40b": "https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json",
    "tiiuae/falcon-7b": "https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json",
}







class __snake_case			(			a		):
   UpperCAmelCase__  :    Optional[Any]          =  '''falcon'''
   UpperCAmelCase__  :    List[Any]          =  ['''past_key_values''']







   def __init__(      self		:						Union[str, Any]					,						_snake_case		:						List[str]=65024					,						_snake_case		:						int=4544					,						_snake_case		:						int=32					,						_snake_case		:						Any=71					,						_snake_case		:						int=1e-5					,						_snake_case		:						Dict=0.0_2					,						_snake_case		:						int=True					,						_snake_case		:						List[Any]=0.0					,						_snake_case		:						Tuple=0.0					,						_snake_case		:						int=None					,						_snake_case		:						Tuple=False					,						_snake_case		:						Any=False					,						_snake_case		:						str=True					,						_snake_case		:						Any=True					,						_snake_case		:						List[str]=False					,						_snake_case		:						Tuple=11					,						_snake_case		:						Dict=11					,						**_snake_case		:						Optional[int]					,						):

        """simple docstring"""


        UpperCAmelCase_          =							vocab_size
        # Backward compatibility with n_embed kwarg
        UpperCAmelCase_          =							kwargs.pop('''n_embed'''					,						_snake_case)
        UpperCAmelCase_          =							hidden_size if n_embed is None else n_embed
        UpperCAmelCase_          =							num_hidden_layers
        UpperCAmelCase_          =							num_attention_heads
        UpperCAmelCase_          =							layer_norm_epsilon
        UpperCAmelCase_          =							initializer_range
        UpperCAmelCase_          =							use_cache
        UpperCAmelCase_          =							hidden_dropout
        UpperCAmelCase_          =							attention_dropout

        UpperCAmelCase_          =							bos_token_id
        UpperCAmelCase_          =							eos_token_id
        UpperCAmelCase_          =							num_attention_heads if num_kv_heads is None else num_kv_heads
        UpperCAmelCase_          =							alibi
        UpperCAmelCase_          =							new_decoder_architecture
        UpperCAmelCase_          =							multi_query  # Ignored when new_decoder_architecture is True
        UpperCAmelCase_          =							parallel_attn
        UpperCAmelCase_          =							bias

        super().__init__(bos_token_id=_snake_case					,						eos_token_id=_snake_case					,						**_snake_case)







   @property
   def lowerCamelCase						(      self		:						List[Any]):

        """simple docstring"""


        return self.hidden_size // self.num_attention_heads







   @property
   def lowerCamelCase						(      self		:						Union[str, Any]):

        """simple docstring"""


        return not self.alibi






