

import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    def __init__(      self :       str,lowercase_ :       NestedDataStructureLike[PathLike],lowercase_ :       Optional[NamedSplit] = None,lowercase_ :       Optional[Features] = None,lowercase_ :       str = None,lowercase_ :       bool = False,lowercase_ :       bool = False,lowercase_ :       Optional[str] = None,lowercase_ :       Optional[int] = None,**lowercase_ :       int,)->    Any:





        '''simple docstring'''







        super().__init__(
            lowercase_,split=lowercase_,features=lowercase_,cache_dir=lowercase_,keep_in_memory=lowercase_,streaming=lowercase_,num_proc=lowercase_,**lowercase_,)
        A__          =   field
        A__          =   path_or_paths if isinstance(lowercase_,lowercase_		) else {self.split: path_or_paths}
        A__          =   Json(
            cache_dir=lowercase_,data_files=lowercase_,features=lowercase_,field=lowercase_,**lowercase_,)


    def 						snake_case__     (      self :       Any		)->    str:





        '''simple docstring'''







        if self.streaming:
            A__          =   self.builder.as_streaming_dataset(split=self.split		)
        # Build regular (map-style) dataset
        else:
            A__          =   None
            A__          =   None
            A__          =   None
            A__          =   None

            self.builder.download_and_prepare(
                download_config=lowercase_,download_mode=lowercase_,verification_mode=lowercase_,base_path=lowercase_,num_proc=self.num_proc,)
            A__          =   self.builder.as_dataset(
                split=self.split,verification_mode=lowercase_,in_memory=self.keep_in_memory		)
        return dataset




class  A						:







    """simple docstring"""





    def __init__(      self :       Tuple,lowercase_ :       Dataset,lowercase_ :       Union[PathLike, BinaryIO],lowercase_ :       Optional[int] = None,lowercase_ :       Optional[int] = None,**lowercase_ :       Tuple,)->    Union[str, Any]:





        '''simple docstring'''







        if num_proc is not None and num_proc <= 0:
            raise ValueError(F'num_proc {num_proc} must be an integer > 0.'		)

        A__          =   dataset
        A__          =   path_or_buf
        A__          =   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        A__          =   num_proc
        A__          =   'utf-8'
        A__          =   to_json_kwargs


    def 						snake_case__     (      self :       List[Any]		)->    int:





        '''simple docstring'''







        A__          =   self.to_json_kwargs.pop('path_or_buf',lowercase_		)
        A__          =   self.to_json_kwargs.pop('orient','records'		)
        A__          =   self.to_json_kwargs.pop('lines',True if orient == 'records' else False		)
        A__          =   self.to_json_kwargs.pop('index',False if orient in ['split', 'table'] else True		)
        A__          =   self.to_json_kwargs.pop('compression',lowercase_		)

        if compression not in [None, "infer", "gzip", "bz2", "xz"]:
            raise NotImplementedError(F'`datasets` currently does not support {compression} compression'		)

        if isinstance(self.path_or_buf,(str, bytes, os.PathLike)		):
            with fsspec.open(self.path_or_buf,'wb',compression=lowercase_		) as buffer:
                A__          =   self._write(file_obj=lowercase_,orient=lowercase_,lines=lowercase_,index=lowercase_,**self.to_json_kwargs		)
        else:
            if compression:
                raise NotImplementedError(
                    F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
                    ' was passed. Please provide a local path instead.'		)
            A__          =   self._write(
                file_obj=self.path_or_buf,orient=lowercase_,lines=lowercase_,index=lowercase_,**self.to_json_kwargs		)
        return written


    def 						snake_case__     (      self :       List[Any],lowercase_ :       int		)->    Dict:





        '''simple docstring'''







        A__      ,						A__      ,						A__      ,						A__      ,						A__          =   args

        A__          =   query_table(
            table=self.dataset.data,key=slice(lowercase_,offset + self.batch_size		),indices=self.dataset._indices,)
        A__          =   batch.to_pandas().to_json(
            path_or_buf=lowercase_,orient=lowercase_,lines=lowercase_,index=lowercase_,**lowercase_		)
        if not json_str.endswith('\n'		):
            json_str += "\n"
        return json_str.encode(self.encoding		)


    def 						snake_case__     (      self :       Any,lowercase_ :       BinaryIO,lowercase_ :       Optional[Any],lowercase_ :       Optional[int],lowercase_ :       Optional[Any],**lowercase_ :       Optional[Any],)->    int:





        '''simple docstring'''







        A__          =   0

        if self.num_proc is None or self.num_proc == 1:
            for offset in logging.tqdm(
                range(0,len(self.dataset		),self.batch_size		),unit='ba',disable=not logging.is_progress_bar_enabled(),desc='Creating json from Arrow format',):
                A__          =   self._batch_json((offset, orient, lines, index, to_json_kwargs)		)
                written += file_obj.write(lowercase_		)
        else:
            A__      ,						A__          =   len(self.dataset		), self.batch_size
            with multiprocessing.Pool(self.num_proc		) as pool:
                for json_str in logging.tqdm(
                    pool.imap(
                        self._batch_json,[(offset, orient, lines, index, to_json_kwargs) for offset in range(0,lowercase_,lowercase_		)],),total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size,unit='ba',disable=not logging.is_progress_bar_enabled(),desc='Creating json from Arrow format',):
                    written += file_obj.write(lowercase_		)

        return written




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_       		=  logging.get_logger(__name__)

lowercase_       		=  {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    lowerCamelCase			=							'open-llama'


    def __init__(      self :       Any,lowercase_ :       Optional[int]=1_0_0_0_0_0,lowercase_ :       Union[str, Any]=4_0_9_6,lowercase_ :       Dict=1_1_0_0_8,lowercase_ :       Dict=3_2,lowercase_ :       Optional[int]=3_2,lowercase_ :       Dict="silu",lowercase_ :       Union[str, Any]=2_0_4_8,lowercase_ :       Optional[int]=0.02,lowercase_ :       Dict=1E-6,lowercase_ :       Dict=True,lowercase_ :       List[Any]=0,lowercase_ :       Optional[int]=1,lowercase_ :       str=2,lowercase_ :       str=False,lowercase_ :       str=True,lowercase_ :       int=0.1,lowercase_ :       List[Any]=0.1,lowercase_ :       List[Any]=True,lowercase_ :       Union[str, Any]=True,lowercase_ :       Any=None,**lowercase_ :       List[Any],)->    Tuple:





        '''simple docstring'''







        A__          =   vocab_size
        A__          =   max_position_embeddings
        A__          =   hidden_size
        A__          =   intermediate_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   hidden_act
        A__          =   initializer_range
        A__          =   rms_norm_eps
        A__          =   use_cache
        A__          =   kwargs.pop(
            'use_memorry_efficient_attention',lowercase_		)
        A__          =   hidden_dropout_prob
        A__          =   attention_dropout_prob
        A__          =   use_stable_embedding
        A__          =   shared_input_output_embedding
        A__          =   rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=lowercase_,bos_token_id=lowercase_,eos_token_id=lowercase_,tie_word_embeddings=lowercase_,**lowercase_,)


    def 						snake_case__     (      self :       str		)->    str:





        '''simple docstring'''







        if self.rope_scaling is None:
            return

        if not isinstance(self.rope_scaling,lowercase_		) or len(self.rope_scaling		) != 2:
            raise ValueError(
                '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                F'got {self.rope_scaling}'		)
        A__          =   self.rope_scaling.get('type',lowercase_		)
        A__          =   self.rope_scaling.get('factor',lowercase_		)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'		)
        if rope_scaling_factor is None or not isinstance(lowercase_,lowercase_		) or rope_scaling_factor <= 1.0:
            raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'		)

