"""simple docstring"""




from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_a      :		Optional[Any]           =   logging.get_logger(__name__)


_a      :		Optional[Any]           =   {
    'Salesforce/codegen-350M-nl': 'https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json',
    'Salesforce/codegen-350M-multi': 'https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json',
    'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json',
    'Salesforce/codegen-2B-nl': 'https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json',
    'Salesforce/codegen-2B-multi': 'https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json',
    'Salesforce/codegen-2B-mono': 'https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json',
    'Salesforce/codegen-6B-nl': 'https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json',
    'Salesforce/codegen-6B-multi': 'https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json',
    'Salesforce/codegen-6B-mono': 'https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json',
    'Salesforce/codegen-16B-nl': 'https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json',
    'Salesforce/codegen-16B-multi': 'https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json',
    'Salesforce/codegen-16B-mono': 'https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json',
}
class 						__A							( SCREAMING_SNAKE_CASE_		):
     _UpperCamelCase							:			Union[str, Any]							=		"codegen"
     _UpperCamelCase							:			Tuple							=		{
         "max_position_embeddings": "n_positions",
         "hidden_size": "n_embd",
         "num_attention_heads": "n_head",
         "num_hidden_layers": "n_layer",
     }






     def __init__(		self				,     a__=50400				,     a__=2048				,     a__=2048				,     a__=4096				,     a__=28				,     a__=16				,     a__=64				,     a__=None				,     a__="gelu_new"				,     a__=0.0				,     a__=0.0				,     a__=0.0				,     a__=1e-5				,     a__=0.0_2				,     a__=True				,     a__=50256				,     a__=50256				,     a__=False				,     **a__				,     ):
       _lowerCAmelCase						:       Dict  			=     vocab_size
       _lowerCAmelCase						:       Optional[Any]  			=     n_ctx
       _lowerCAmelCase						:       Dict  			=     n_positions
       _lowerCAmelCase						:       Any  			=     n_embd
       _lowerCAmelCase						:       int  			=     n_layer
       _lowerCAmelCase						:       Any  			=     n_head
       _lowerCAmelCase						:       List[Any]  			=     n_inner
       _lowerCAmelCase						:       int  			=     rotary_dim
       _lowerCAmelCase						:       List[str]  			=     activation_function
       _lowerCAmelCase						:       List[str]  			=     resid_pdrop
       _lowerCAmelCase						:       Union[str, Any]  			=     embd_pdrop
       _lowerCAmelCase						:       List[str]  			=     attn_pdrop
       _lowerCAmelCase						:       Union[str, Any]  			=     layer_norm_epsilon
       _lowerCAmelCase						:       Any  			=     initializer_range
       _lowerCAmelCase						:       Union[str, Any]  			=     use_cache

       _lowerCAmelCase						:       Union[str, Any]  			=     bos_token_id
       _lowerCAmelCase						:       Optional[int]  			=     eos_token_id

       super().__init__(
           bos_token_id=a__				,     eos_token_id=a__				,     tie_word_embeddings=a__				,     **a__  )







class 						__A							( SCREAMING_SNAKE_CASE_		):






     def __init__(		self				,     a__				,     a__ = "default"				,     a__ = None				,     a__ = False				,     ):
       super().__init__(a__				,     task=a__				,     patching_specs=a__				,     use_past=a__  )
       if not getattr(self._config				,     """pad_token_id"""				,     a__  ):
         # TODO: how to do that better?
         _lowerCAmelCase						:       List[str]  			=     0






     @property
     def     __A  (		self  ):
       _lowerCAmelCase						:       Tuple  			=     OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}  )
       if self.use_past:
         self.fill_with_past_key_values_(a__				,     direction="""inputs"""  )
         _lowerCAmelCase						:       str  			=     {0: """batch""", 1: """past_sequence + sequence"""}
       else:
         _lowerCAmelCase						:       List[Any]  			=     {0: """batch""", 1: """sequence"""}

       return common_inputs






     @property
     def     __A  (		self  ):
       return self._config.n_layer






     @property
     def     __A  (		self  ):
       return self._config.n_head






     def     __A  (		self				,     a__				,     a__ = -1				,     a__ = -1				,     a__ = False				,     a__ = None				,     ):
       _lowerCAmelCase						:       Any  			=     super(a__				,     self  ).generate_dummy_inputs(
           a__				,     batch_size=a__				,     seq_length=a__				,     is_pair=a__				,     framework=a__  )

       # We need to order the input in the way they appears in the forward()
       _lowerCAmelCase						:       Optional[int]  			=     OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}  )

       # Need to add the past_keys
       if self.use_past:
         if not is_torch_available():
           raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""  )
         else:
           import torch

           _lowerCAmelCase			,	_lowerCAmelCase						:       Tuple  			=     common_inputs["""input_ids"""].shape
           # Not using the same length for past_key_values
           _lowerCAmelCase						:       Any  			=     seqlen + 2
           _lowerCAmelCase						:       int  			=     (
               batch,
               self.num_attention_heads,
               past_key_values_length,
               self._config.hidden_size // self.num_attention_heads,
           )
           _lowerCAmelCase						:       Any  			=     [
               (torch.zeros(a__  ), torch.zeros(a__  )) for _ in range(self.num_layers  )
           ]

       _lowerCAmelCase						:       Union[str, Any]  			=     common_inputs["""attention_mask"""]
       if self.use_past:
         _lowerCAmelCase						:       str  			=     ordered_inputs["""attention_mask"""].dtype
         _lowerCAmelCase						:       int  			=     torch.cat(
             [ordered_inputs["""attention_mask"""], torch.ones(a__				,     a__				,     dtype=a__  )]				,     dim=1  )

       return ordered_inputs






     @property
     def     __A  (		self  ):
       return 13





"""simple docstring"""




import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_a      :		Dict           =   datasets.utils.logging.get_logger(__name__)
@dataclass
class 						__A							( datasets.BuilderConfig		):
     _UpperCamelCase							:			int							=		10_000
     _UpperCamelCase							:			Optional[List[str]]							=		None
     _UpperCamelCase							:			Optional[datasets.Features]							=		None







class 						__A							( datasets.ArrowBasedBuilder		):
     _UpperCamelCase							:			List[str]							=		ParquetConfig






     def     __A  (		self  ):
       return datasets.DatasetInfo(features=self.config.features  )






     def     __A  (		self				,     a__  ):
       if not self.config.data_files:
         raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"  )
       _lowerCAmelCase						:       Optional[Any]  			=     dl_manager.download_and_extract(self.config.data_files  )
       if isinstance(a__				,     (str, list, tuple)  ):
         _lowerCAmelCase						:       Any  			=     data_files
         if isinstance(a__				,     a__  ):
           _lowerCAmelCase						:       Tuple  			=     [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase						:       Any  			=     [dl_manager.iter_files(a__  ) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,     gen_kwargs={"""files""": files}  )]
       _lowerCAmelCase						:       Optional[Any]  			=     []
       for split_name, files in data_files.items():
         if isinstance(a__				,     a__  ):
           _lowerCAmelCase						:       Dict  			=     [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _lowerCAmelCase						:       Tuple  			=     [dl_manager.iter_files(a__  ) for file in files]
         # Infer features is they are stoed in the arrow schema
         if self.info.features is None:
           for file in itertools.chain.from_iterable(a__  ):
             with open(a__				,     """rb"""  ) as f:
               _lowerCAmelCase						:       Optional[Any]  			=     datasets.Features.from_arrow_schema(pq.read_schema(a__  )  )
             break
         splits.append(datasets.SplitGenerator(name=a__				,     gen_kwargs={"""files""": files}  )  )
       return splits






     def     __A  (		self				,     a__  ):
       if self.info.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _lowerCAmelCase						:       Optional[int]  			=     table_cast(a__				,     self.info.features.arrow_schema  )
       return pa_table






     def     __A  (		self				,     a__  ):
       _lowerCAmelCase						:       Optional[int]  			=     self.info.features.arrow_schema if self.info.features is not None else None
       if self.info.features is not None and self.config.columns is not None:
         if sorted(field.name for field in schema  ) != sorted(self.config.columns  ):
           raise ValueError(
               F"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"  )
       for file_idx, file in enumerate(itertools.chain.from_iterable(a__  )  ):
         with open(a__				,     """rb"""  ) as f:
           _lowerCAmelCase						:       Tuple  			=     pq.ParquetFile(a__  )
           try:
             for batch_idx, record_batch in enumerate(
                 parquet_file.iter_batches(batch_size=self.config.batch_size				,     columns=self.config.columns  )  ):
               _lowerCAmelCase						:       Any  			=     pa.Table.from_batches([record_batch]  )
               # Uncomment for debugging (will print the Arrow table size and elements)
               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
               yield F"{file_idx}_{batch_idx}", self._cast_table(a__  )
           except ValueError as e:
             logger.error(F"Failed to read file '{file}' with error {type(a__  )}: {e}"  )
             raise




