





import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


_UpperCAmelCase										=					datasets.utils.logging.get_logger(__name__)



@dataclass
class 				_UpperCamelCase       (						datasets.BuilderConfig  ):
       _UpperCamelCase					:       Optional[datasets.Features]       = None



class 				_UpperCamelCase       (						datasets.ArrowBasedBuilder  ):
       _UpperCamelCase					:       Union[str, Any]       = ArrowConfig

       def lowercase			(		self:			Optional[int]		)							->			Any:




              """simple docstring"""


              return datasets.DatasetInfo(features=self.config.features		)

       def lowercase			(		self:			List[Any]       , _SCREAMING_SNAKE_CASE:			List[str]		)							->			int:




              """simple docstring"""


              if not self.config.data_files:
                     raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
              UpperCamelCase_ 							=	dl_manager.download_and_extract(self.config.data_files		)
              if isinstance(_SCREAMING_SNAKE_CASE       , (str, list, tuple)		):
                     UpperCamelCase_ 							=	data_files
                     if isinstance(_SCREAMING_SNAKE_CASE       , _SCREAMING_SNAKE_CASE		):
                            UpperCamelCase_ 							=	[files]
                     # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                     UpperCamelCase_ 							=	[dl_manager.iter_files(_SCREAMING_SNAKE_CASE		) for file in files]
                     return [datasets.SplitGenerator(name=datasets.Split.TRAIN       , gen_kwargs={"files": files}		)]
              UpperCamelCase_ 							=	[]
              for split_name, files in data_files.items():
                     if isinstance(_SCREAMING_SNAKE_CASE       , _SCREAMING_SNAKE_CASE		):
                            UpperCamelCase_ 							=	[files]
                     # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                     UpperCamelCase_ 							=	[dl_manager.iter_files(_SCREAMING_SNAKE_CASE		) for file in files]
                     # Infer features is they are stoed in the arrow schema
                     if self.info.features is None:
                            for file in itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE		):
                                   with open(_SCREAMING_SNAKE_CASE       , "rb"		) as f:
                                          UpperCamelCase_ 							=	datasets.Features.from_arrow_schema(pa.ipc.open_stream(_SCREAMING_SNAKE_CASE		).schema		)
                                   break
                     splits.append(datasets.SplitGenerator(name=_SCREAMING_SNAKE_CASE       , gen_kwargs={"files": files}		)		)
              return splits

       def lowercase			(		self:			Optional[int]       , _SCREAMING_SNAKE_CASE:			pa.Table		)							->			pa.Table:




              """simple docstring"""


              if self.info.features is not None:
                     # more expensive cast to support nested features with keys in a different order
                     # allows str <-> int/float or str to Audio for example
                     UpperCamelCase_ 							=	table_cast(_SCREAMING_SNAKE_CASE       , self.info.features.arrow_schema		)
              return pa_table



       def lowercase			(		self:			List[str]       , _SCREAMING_SNAKE_CASE:			Optional[int]		)							->			Dict:




              """simple docstring"""


              for file_idx, file in enumerate(itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE		)		):
                     with open(_SCREAMING_SNAKE_CASE       , "rb"		) as f:
                            try:
                                   for batch_idx, record_batch in enumerate(pa.ipc.open_stream(_SCREAMING_SNAKE_CASE		)		):
                                          UpperCamelCase_ 							=	pa.Table.from_batches([record_batch]		)
                                          # Uncomment for debugging (will print the Arrow table size and elements)
                                          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                          yield f'''{file_idx}_{batch_idx}''', self._cast_table(_SCREAMING_SNAKE_CASE		)
                            except ValueError as e:
                                   logger.error(f'''Failed to read file \'{file}\' with error {type(_SCREAMING_SNAKE_CASE		)}: {e}'''		)
                                   raise



import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


_A       =					logging.getLogger(__name__)







@dataclass
class 	lowerCamelCase						:
     UpperCAmelCase__							:				str 	=						field(
         metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}    )
     UpperCAmelCase__							:				Optional[str] 	=						field(
         default=A_				,					metadata={"help": "Pretrained config name or path if not the same as model_name"}    )
     UpperCAmelCase__							:				Optional[str] 	=						field(
         default=A_				,					metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}    )
     UpperCAmelCase__							:				bool 	=						field(default=A_				,					metadata={"help": "Set this flag to use fast tokenization."}    )
     # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
     # or just modify its tokenizer_config.json.
     UpperCAmelCase__							:				Optional[str] 	=						field(
         default=A_				,					metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}				,					)



def 		lowercase_     (  )				->				int:
    """simple docstring"""

    snake_case						=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)						)

    if len(sys.argv						) == 2 and sys.argv[1].endswith(".json"						):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        snake_case	,				snake_case	,				snake_case						=   parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]						)						)
    else:
        snake_case	,				snake_case	,				snake_case						=   parser.parse_args_into_dataclasses()

    if (
        os.path.exists(training_args.output_dir						)
        and os.listdir(training_args.output_dir						)
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
        raise ValueError(
            F'Output directory ({training_args.output_dir}) already exists and is not empty. Use'
            " --overwrite_output_dir to overcome."						)

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"					,						datefmt="%m/%d/%Y %H:%M:%S"					,						level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN					,						)
    logger.warning(
        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"					,						training_args.local_rank					,						training_args.device					,						training_args.n_gpu					,						bool(training_args.local_rank != -1						)					,						training_args.fpaa					,						)
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank						):
        transformers.utils.logging.set_verbosity_info()
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
    logger.info("Training/evaluation parameters %s"					,						A__						)

    # Prepare Question-Answering task
    # Load pretrained model and tokenizer
    #
    # Distributed training:
    # The .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.

    snake_case						=   AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path					,						cache_dir=model_args.cache_dir					,						)
    snake_case						=   AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path					,						cache_dir=model_args.cache_dir					,						use_fast=A__					,						)
    snake_case						=   AutoModelForQuestionAnswering.from_pretrained(
        model_args.model_name_or_path					,						from_tf=bool(".ckpt" in model_args.model_name_or_path						)					,						config=A__					,						cache_dir=model_args.cache_dir					,						)

    # Get datasets
    snake_case						=   hasattr(model.config					,						"lang2id"						)
    snake_case						=   (
        SquadDataset(
            A__					,						tokenizer=A__					,						is_language_sensitive=A__					,						cache_dir=model_args.cache_dir						)
        if training_args.do_train
        else None
    )
    snake_case						=   (
        SquadDataset(
            A__					,						tokenizer=A__					,						mode="dev"					,						is_language_sensitive=A__					,						cache_dir=model_args.cache_dir					,						)
        if training_args.do_eval
        else None
    )

    # Data collator
    snake_case						=   DataCollatorWithPadding(A__					,						pad_to_multiple_of=8						) if training_args.fpaa else None

    # Initialize our Trainer
    snake_case						=   Trainer(
        model=A__					,						args=A__					,						train_dataset=A__					,						eval_dataset=A__					,						data_collator=A__					,						)

    # Training
    if training_args.do_train:
        trainer.train(
            model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path						) else None						)
        trainer.save_model()
        # For convenience, we also re-save the tokenizer to the same directory,
        # so that you can share your model easily on huggingface.co/models =)
        if trainer.is_world_master():
            tokenizer.save_pretrained(training_args.output_dir						)



def 		lowercase_     (  A__						)				->				Optional[Any]:
    """simple docstring"""

    main()


if __name__ == "__main__":
   main()

