



'''simple docstring'''







import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


lowercase__  :	Dict            =      logging.getLogger(__name__)

require_version('pytorch_lightning>=1.0.4')

lowercase__  :	List[Any]            =      {
    'base': AutoModel,
    'sequence-classification': AutoModelForSequenceClassification,
    'question-answering': AutoModelForQuestionAnswering,
    'pretraining': AutoModelForPreTraining,
    'token-classification': AutoModelForTokenClassification,
    'language-modeling': AutoModelWithLMHead,
    'summarization': AutoModelForSeqaSeqLM,
    'translation': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
lowercase__  :	Dict            =      {
    'linear': get_linear_schedule_with_warmup,
    'cosine': get_cosine_schedule_with_warmup,
    'cosine_w_restarts': get_cosine_with_hard_restarts_schedule_with_warmup,
    'polynomial': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
lowercase__  :	Tuple            =      sorted(arg_to_scheduler.keys())
lowercase__  :	str            =      '{' + ', '.join(arg_to_scheduler_choices) + '}'

class    lowerCamelCase_				(						pl.LightningModule	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None  ,    __UpperCamelCase="base"  ,    __UpperCamelCase=None  ,    __UpperCamelCase=None  ,    __UpperCamelCase=None  ,    **__UpperCamelCase  ,    ):
       super().__init__()
       # TODO: move to self.save_hyperparameters()
       # self.save_hyperparameters()
       # can also expand arguments into trainer signature for easier reading

       self.save_hyperparameters(__UpperCamelCase			)
       UpperCamelCase__	     =      0
       UpperCamelCase__	     =      Path(self.hparams.output_dir			)
       UpperCamelCase__	     =      self.hparams.cache_dir if self.hparams.cache_dir else None
       if config is None:
           UpperCamelCase__	     =      AutoConfig.from_pretrained(
               self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path  ,    **({"num_labels": num_labels} if num_labels is not None else {})  ,    cache_dir=__UpperCamelCase  ,    **__UpperCamelCase  ,    )
       else:
           UpperCamelCase__	     =      config

       UpperCamelCase__	     =      ("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
       for p in extra_model_params:
           if getattr(self.hparams  ,    __UpperCamelCase  ,    __UpperCamelCase			):
               assert hasattr(self.config  ,    __UpperCamelCase			), f'model config doesn\'t have a `{p}` attribute'
               setattr(self.config  ,    __UpperCamelCase  ,    getattr(self.hparams  ,    __UpperCamelCase			)			)

       if tokenizer is None:
           UpperCamelCase__	     =      AutoTokenizer.from_pretrained(
               self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path  ,    cache_dir=__UpperCamelCase  ,    )
       else:
           UpperCamelCase__	     =      tokenizer
       UpperCamelCase__	     =      MODEL_MODES[mode]
       if model is None:
           UpperCamelCase__	     =      self.model_type.from_pretrained(
               self.hparams.model_name_or_path  ,    from_tf=bool(".ckpt" in self.hparams.model_name_or_path			)  ,    config=self.config  ,    cache_dir=__UpperCamelCase  ,    )
       else:
           UpperCamelCase__	     =      model


   def 						_UpperCamelCase							(	self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       UpperCamelCase__	     =      self.model_type.from_pretrained(*__UpperCamelCase  ,    **__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      arg_to_scheduler[self.hparams.lr_scheduler]
       UpperCamelCase__	     =      get_schedule_func(
           self.opt  ,    num_warmup_steps=self.hparams.warmup_steps  ,    num_training_steps=self.total_steps()			)
       UpperCamelCase__	     =      {"scheduler": scheduler, "interval": "step", "frequency": 1}
       return scheduler


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.model
       UpperCamelCase__	     =      ["bias", "LayerNorm.weight"]
       UpperCamelCase__	     =      [
           {
               "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay			)],
               "weight_decay": self.hparams.weight_decay,
           },
           {
               "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay			)],
               "weight_decay": 0.0,
           },
       ]
       if self.hparams.adafactor:
           UpperCamelCase__	     =      Adafactor(
               __UpperCamelCase  ,    lr=self.hparams.learning_rate  ,    scale_parameter=__UpperCamelCase  ,    relative_step=__UpperCamelCase			)

       else:
           UpperCamelCase__	     =      AdamW(
               __UpperCamelCase  ,    lr=self.hparams.learning_rate  ,    eps=self.hparams.adam_epsilon			)
       UpperCamelCase__	     =      optimizer

       UpperCamelCase__	     =      self.get_lr_scheduler()

       return [optimizer], [scheduler]


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       return self.validation_step(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       return self.validation_end(__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      max(1  ,    self.hparams.gpus			)  # TODO: consider num_tpu_cores
       UpperCamelCase__	     =      self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
       return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       if mode == "test":
           UpperCamelCase__	     =      len(self.test_dataloader().dataset			)
       else:
           UpperCamelCase__	     =      self.get_dataloader("train"  ,    self.hparams.train_batch_size  ,    shuffle=__UpperCamelCase			)
           UpperCamelCase__	     =      len(self.train_dataloader().dataset			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase = False			):
       raise NotImplementedError("You must implement this for your task"			)


   def 						_UpperCamelCase							(	self			):
       return self.train_loader


   def 						_UpperCamelCase							(	self			):
       return self.get_dataloader("dev"  ,    self.hparams.eval_batch_size  ,    shuffle=__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       return self.get_dataloader("test"  ,    self.hparams.eval_batch_size  ,    shuffle=__UpperCamelCase			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       return os.path.join(
           self.hparams.data_dir  ,    "cached_{}_{}_{}".format(
               __UpperCamelCase  ,    list(filter(__UpperCamelCase  ,    self.hparams.model_name_or_path.split("/"			)			)			).pop()  ,    str(self.hparams.max_seq_length			)  ,    )  ,    )


   @pl.utilities.rank_zero_only
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      self.output_dir.joinpath("best_tfmr"			)
       UpperCamelCase__	     =      self.step_count
       self.model.save_pretrained(__UpperCamelCase			)
       self.tokenizer.save_pretrained(__UpperCamelCase			)







   @staticmethod
   def 						_UpperCamelCase							(	__UpperCamelCase  ,    __UpperCamelCase			):
       parser.add_argument(
           "--model_name_or_path"  ,    default=__UpperCamelCase  ,    type=__UpperCamelCase  ,    required=__UpperCamelCase  ,    help="Path to pretrained model or model identifier from huggingface.co/models"  ,    )
       parser.add_argument(
           "--config_name"  ,    default=""  ,    type=__UpperCamelCase  ,    help="Pretrained config name or path if not the same as model_name"			)
       parser.add_argument(
           "--tokenizer_name"  ,    default=__UpperCamelCase  ,    type=__UpperCamelCase  ,    help="Pretrained tokenizer name or path if not the same as model_name"  ,    )
       parser.add_argument(
           "--cache_dir"  ,    default=""  ,    type=__UpperCamelCase  ,    help="Where do you want to store the pre-trained models downloaded from huggingface.co"  ,    )
       parser.add_argument(
           "--encoder_layerdrop"  ,    type=__UpperCamelCase  ,    help="Encoder layer dropout probability (Optional). Goes into model.config"  ,    )
       parser.add_argument(
           "--decoder_layerdrop"  ,    type=__UpperCamelCase  ,    help="Decoder layer dropout probability (Optional). Goes into model.config"  ,    )
       parser.add_argument(
           "--dropout"  ,    type=__UpperCamelCase  ,    help="Dropout probability (Optional). Goes into model.config"  ,    )
       parser.add_argument(
           "--attention_dropout"  ,    type=__UpperCamelCase  ,    help="Attention dropout probability (Optional). Goes into model.config"  ,    )
       parser.add_argument("--learning_rate"  ,    default=5e-5  ,    type=__UpperCamelCase  ,    help="The initial learning rate for Adam."			)
       parser.add_argument(
           "--lr_scheduler"  ,    default="linear"  ,    choices=__UpperCamelCase  ,    metavar=__UpperCamelCase  ,    type=__UpperCamelCase  ,    help="Learning rate scheduler"  ,    )
       parser.add_argument("--weight_decay"  ,    default=0.0  ,    type=__UpperCamelCase  ,    help="Weight decay if we apply some."			)
       parser.add_argument("--adam_epsilon"  ,    default=1e-8  ,    type=__UpperCamelCase  ,    help="Epsilon for Adam optimizer."			)
       parser.add_argument("--warmup_steps"  ,    default=0  ,    type=__UpperCamelCase  ,    help="Linear warmup over warmup_steps."			)
       parser.add_argument("--num_workers"  ,    default=4  ,    type=__UpperCamelCase  ,    help="kwarg passed to DataLoader"			)
       parser.add_argument("--num_train_epochs"  ,    dest="max_epochs"  ,    default=3  ,    type=__UpperCamelCase			)
       parser.add_argument("--train_batch_size"  ,    default=32  ,    type=__UpperCamelCase			)
       parser.add_argument("--eval_batch_size"  ,    default=32  ,    type=__UpperCamelCase			)
       parser.add_argument("--adafactor"  ,    action="store_true"			)

class    lowerCamelCase_				(						pl.Callback	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      trainer.lr_schedulers[0]["scheduler"]
       UpperCamelCase__	     =      {f'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()			)}
       pl_module.logger.log_metrics(__UpperCamelCase			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       rank_zero_info("***** Validation results *****"			)
       UpperCamelCase__	     =      trainer.callback_metrics
       # Log results
       for key in sorted(__UpperCamelCase			):
           if key not in ["log", "progress_bar"]:
               rank_zero_info("{} = {}\n".format(__UpperCamelCase  ,    str(metrics[key]			)			)			)







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase			):
       rank_zero_info("***** Test results *****"			)
       UpperCamelCase__	     =      trainer.callback_metrics
       # Log and save results to file
       UpperCamelCase__	     =      os.path.join(pl_module.hparams.output_dir  ,    "test_results.txt"			)
       with open(__UpperCamelCase  ,    "w"			) as writer:
           for key in sorted(__UpperCamelCase			):
               if key not in ["log", "progress_bar"]:
                   rank_zero_info("{} = {}\n".format(__UpperCamelCase  ,    str(metrics[key]			)			)			)
                   writer.write("{} = {}\n".format(__UpperCamelCase  ,    str(metrics[key]			)			)			)





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_						):
    #  To allow all pl args uncomment the following line
    #  parser = pl.Trainer.add_argparse_args(parser)
    parser.add_argument(
        "--output_dir"       ,	default=lowerCAmelCase_       ,	type=lowerCAmelCase_       ,	required=lowerCAmelCase_       ,	help="The output directory where the model predictions and checkpoints will be written."       ,	)
    parser.add_argument(
        "--fp16"       ,	action="store_true"       ,	help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"       ,	)

    parser.add_argument(
        "--fp16_opt_level"       ,	type=lowerCAmelCase_       ,	default="O2"       ,	help=(
            "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
            "See details at https://nvidia.github.io/apex/amp.html"
        )       ,	)
    parser.add_argument("--n_tpu_cores"       ,	dest="tpu_cores"       ,	type=lowerCAmelCase_						)
    parser.add_argument("--max_grad_norm"       ,	dest="gradient_clip_val"       ,	default=1.0       ,	type=lowerCAmelCase_       ,	help="Max gradient norm"						)
    parser.add_argument("--do_train"       ,	action="store_true"       ,	help="Whether to run training."						)
    parser.add_argument("--do_predict"       ,	action="store_true"       ,	help="Whether to run predictions on the test set."						)
    parser.add_argument(
        "--gradient_accumulation_steps"       ,	dest="accumulate_grad_batches"       ,	type=lowerCAmelCase_       ,	default=1       ,	help="Number of updates steps to accumulate before performing a backward/update pass."       ,	)
    parser.add_argument("--seed"       ,	type=lowerCAmelCase_       ,	default=42       ,	help="random seed for initialization"						)
    parser.add_argument(
        "--data_dir"       ,	default=lowerCAmelCase_       ,	type=lowerCAmelCase_       ,	required=lowerCAmelCase_       ,	help="The input data dir. Should contain the training files for the CoNLL-2003 NER task."       ,	)





def     snake_case__			(lowerCAmelCase_       ,	lowerCAmelCase_       ,	lowerCAmelCase_=None       ,	lowerCAmelCase_=True       ,	lowerCAmelCase_=[]       ,	lowerCAmelCase_=None       ,	lowerCAmelCase_=None       ,	**lowerCAmelCase_       ,	):
    pl.seed_everything(args.seed						)

    # init model
    UpperCamelCase__	     =      Path(model.hparams.output_dir						)
    odir.mkdir(exist_ok=lowerCAmelCase_						)

    # add custom checkpoints
    if checkpoint_callback is None:
        UpperCamelCase__	     =      pl.callbacks.ModelCheckpoint(
            filepath=args.output_dir       ,	prefix="checkpoint"       ,	monitor="val_loss"       ,	mode="min"       ,	save_top_k=1						)
    if early_stopping_callback:
        extra_callbacks.append(lowerCAmelCase_						)
    if logging_callback is None:
        UpperCamelCase__	     =      LoggingCallback()

    UpperCamelCase__	     =      {}

    # TODO: remove with PyTorch 1.6 since pl uses native amp
    if args.fpaa:
        UpperCamelCase__	     =      16
        UpperCamelCase__	     =      args.fpaa_opt_level

    if args.gpus > 1:
        UpperCamelCase__	     =      "ddp"

    UpperCamelCase__	     =      args.accumulate_grad_batches
    UpperCamelCase__	     =      extra_train_kwargs.get("accelerator"       ,	lowerCAmelCase_						)
    UpperCamelCase__	     =      extra_train_kwargs.get("profiler"       ,	lowerCAmelCase_						)

    UpperCamelCase__	     =      pl.Trainer.from_argparse_args(
        lowerCAmelCase_       ,	weights_summary=lowerCAmelCase_       ,	callbacks=[logging_callback] + extra_callbacks       ,	logger=lowerCAmelCase_       ,	checkpoint_callback=lowerCAmelCase_       ,	**lowerCAmelCase_       ,	)

    if args.do_train:
        trainer.fit(lowerCAmelCase_						)

    return trainer









'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      foo







   def __call__(	self			):
       return self.foo

class    lowerCamelCase_				(						datasets.Dataset	):






   '''simple docstring'''

   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      callable


   def __call__(	self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       if self.callable is not None:
           return self.callable(*__UpperCamelCase  ,    **__UpperCamelCase			)







   def __getstate__(	self			):
       raise pickle.PicklingError()

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       def encode(__UpperCamelCase			):
           return tokenizer(__UpperCamelCase			)

       # TODO: add hash consistency tests across sessions
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-cased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       tokenizer("Hello world !"			)  # call once to change the tokenizer's cache
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_regex
   def 						_UpperCamelCase							(	self			):
       import regex

       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("bar"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       def func():
           return foo

       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()

       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      (
           "co_argcount",
           "co_kwonlyargcount",
           "co_nlocals",
           "co_stacksize",
           "co_flags",
           "co_code",
           "co_consts",
           "co_names",
           "co_varnames",
           "co_filename",
           "co_name",
           "co_firstlineno",
           "co_lnotab",
           "co_freevars",
           "co_cellvars",
       )

       def _create_code_pyaa(*__UpperCamelCase			):
           if hasattr(args[-3]  ,    "encode"			):
               UpperCamelCase__	     =      list(__UpperCamelCase			)
               UpperCamelCase__	     =      args[-3].encode()  # co_lnotab
               UpperCamelCase__	     =      args[-10].encode()  # co_code
           if hasattr(__UpperCamelCase  ,    "co_posonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(*__UpperCamelCase			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(args[0]  ,    0  ,    *args[1:]			)
               return CodeType(args[0]  ,    0  ,    0  ,    *args[1:]			)
           elif hasattr(__UpperCamelCase  ,    "co_kwonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(args[0]  ,    *args[2:]			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(*__UpperCamelCase			)
               return CodeType(args[0]  ,    0  ,    *args[1:]			)
           if len(__UpperCamelCase			) == 16:
               return CodeType(args[0]  ,    *args[3:]			)
           elif len(__UpperCamelCase			) == 15:
               return CodeType(args[0]  ,    *args[2:]			)
           return CodeType(*__UpperCamelCase			)

       def create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			):
           def func():
               return returned_obj

           UpperCamelCase__	     =      func.__code__
           # Use _create_code from dill in order to make it work for different python versions
           if datasets.config.PY_VERSION < version.parse("3.8"			):
               UpperCamelCase__	     =      _create_code_pyaa(
                   *[getattr(__UpperCamelCase  ,    __UpperCamelCase			) if k != "co_filename" else co_filename for k in code_args_pyaa]			)
           else:
               UpperCamelCase__	     =      code.replace(co_filename=__UpperCamelCase			)
           return FunctionType(__UpperCamelCase  ,    func.__globals__  ,    func.__name__  ,    func.__defaults__  ,    func.__closure__			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-5-713f6613acf3>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "654123987.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      [0], [1]

       def func():
           return foo, bar

       UpperCamelCase__	     =      "__main__"

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"foo": foo, "bar": bar}

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"bar": bar, "foo": foo}

       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("there"			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("there"			)			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["there", Foo("there"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       with self.assertRaises(pickle.PicklingError			):
           Hasher.hash(UnpicklableCallable(Foo("hello"			)			)			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "abc"
       UpperCamelCase__	     =      [string, string]  # two strings have the same ids
       UpperCamelCase__	     =      [string, string]
       UpperCamelCase__	     =      json.loads(f'["{string}", "{string}"]'			)  # two strings have different ids
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIsNot(obja[0]  ,    __UpperCamelCase			)
       self.assertIsNot(obja[0]  ,    obja[1]			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_tiktoken
   def 						_UpperCamelCase							(	self			):
       import tiktoken

       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("r50k_base"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_torch
   def 						_UpperCamelCase							(	self			):
       import torch

       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([2.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_spacy
   @require_spacy_model("en_core_web_sm"			)
   @require_spacy_model("fr_core_news_sm"			)
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       import spacy

       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("fr_core_news_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)





@pytest.mark.integration
def     snake_case__			(lowerCAmelCase_						):
    UpperCamelCase__	     =      tmp_path / "dir1"
    UpperCamelCase__	     =      tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    UpperCamelCase__	     =      "script.py"
    UpperCamelCase__	     =      dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
    UpperCamelCase__	     =      dira / script_filename
    UpperCamelCase__	     =      dira / script_filename
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    assert fingerprinta == fingerprinta





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}
    UpperCamelCase__	     =      DatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						)
    UpperCamelCase__	     =      dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2						) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="1.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="2.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3





def     snake_case__			():
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    UpperCamelCase__	     =      Hasher()
    hasher.update(lambda lowerCAmelCase_						: x						)




