


import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


SCREAMING_SNAKE_CASE__       			=				logging.get_logger(__name__)







@dataclass
class     __lowerCamelCase     :


							"""simple docstring"""

							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				None
							lowerCAmelCase__          =				None




							def 				A__      (  self  )     ->       Tuple:






								'''simple docstring'''






								return json.dumps(dataclasses.asdict(self  )						,       indent=2  ) + "\n"







@dataclass(frozen=lowerCAmelCase_     )
class     __lowerCamelCase     :


							"""simple docstring"""

							lowerCAmelCase__          =				42
							lowerCAmelCase__          =				None
							lowerCAmelCase__          =				None
							lowerCAmelCase__          =				None




							def 				A__      (  self  )     ->       Optional[int]:






								'''simple docstring'''






								return json.dumps(dataclasses.asdict(self  )  ) + "\n"







class     __lowerCamelCase     :


							"""simple docstring"""

							def 				A__      (  self						,       UpperCAmelCase  )     ->       int:






								'''simple docstring'''






								raise NotImplementedError()




							def 				A__      (  self						,       UpperCAmelCase  )     ->       Any:






								'''simple docstring'''






								raise NotImplementedError()




							def 				A__      (  self						,       UpperCAmelCase  )     ->       Optional[Any]:






								'''simple docstring'''






								raise NotImplementedError()




							def 				A__      (  self						,       UpperCAmelCase  )     ->       Optional[Any]:






								'''simple docstring'''






								raise NotImplementedError()




							def 				A__      (  self  )     ->       int:






								'''simple docstring'''






								raise NotImplementedError()




							def 				A__      (  self						,       UpperCAmelCase  )     ->       Tuple:






								'''simple docstring'''






								if len(self.get_labels()  ) > 1:
									lowercase_			 = self.get_labels()[int(example.label  )]
								return example




							@classmethod
							def 				A__      (  cls						,       UpperCAmelCase						,       UpperCAmelCase=None  )     ->       Dict:






								'''simple docstring'''






								with open(__UpperCamelCase						,       "r"						,       encoding="utf-8-sig"  ) as f:
									return list(csv.reader(__UpperCamelCase						,       delimiter="\t"						,       quotechar=__UpperCamelCase  )  )







class     __lowerCamelCase     (  lowerCAmelCase_     ):


							"""simple docstring"""

							def __init__(  self						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase="classification"						,       UpperCAmelCase=False  )     ->       Dict:






								'''simple docstring'''






								lowercase_			 = [] if labels is None else labels
								lowercase_			 = [] if examples is None else examples
								lowercase_			 = mode
								lowercase_			 = verbose




							def __len__(  self  )     ->       Optional[int]:






								'''simple docstring'''






								return len(self.examples  )




							def __getitem__(  self						,       UpperCAmelCase  )     ->       Dict:






								'''simple docstring'''






								if isinstance(__UpperCamelCase						,       __UpperCamelCase  ):
									return SingleSentenceClassificationProcessor(labels=self.labels						,       examples=self.examples[idx]  )
								return self.examples[idx]




							@classmethod
							def 				A__      (  cls						,       UpperCAmelCase						,       UpperCAmelCase=""						,       UpperCAmelCase=0						,       UpperCAmelCase=1						,       UpperCAmelCase=None						,       UpperCAmelCase=False						,       **UpperCAmelCase  )     ->       Optional[int]:






								'''simple docstring'''






								lowercase_			 = cls(**__UpperCamelCase  )
								processor.add_examples_from_csv(
								    __UpperCamelCase						,       split_name=__UpperCamelCase						,       column_label=__UpperCamelCase						,       column_text=__UpperCamelCase						,       column_id=__UpperCamelCase						,       skip_first_row=__UpperCamelCase						,       overwrite_labels=__UpperCamelCase						,       overwrite_examples=__UpperCamelCase						,       )
								return processor




							@classmethod
							def 				A__      (  cls						,       UpperCAmelCase						,       UpperCAmelCase=None						,       **UpperCAmelCase  )     ->       Dict:






								'''simple docstring'''






								lowercase_			 = cls(**__UpperCamelCase  )
								processor.add_examples(__UpperCamelCase						,       labels=__UpperCamelCase  )
								return processor




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase=""						,       UpperCAmelCase=0						,       UpperCAmelCase=1						,       UpperCAmelCase=None						,       UpperCAmelCase=False						,       UpperCAmelCase=False						,       UpperCAmelCase=False						,       )     ->       Tuple:






								'''simple docstring'''






								lowercase_			 = self._read_tsv(__UpperCamelCase  )
								if skip_first_row:
									lowercase_			 = lines[1:]
								lowercase_			 = []
								lowercase_			 = []
								lowercase_			 = []
								for i, line in enumerate(__UpperCamelCase  ):
									texts.append(line[column_text]  )
									labels.append(line[column_label]  )
									if column_id is not None:
										ids.append(line[column_id]  )
									else:
										lowercase_			 = F'{split_name}-{i}' if split_name else str(__UpperCamelCase  )
										ids.append(__UpperCamelCase  )

								return self.add_examples(
								    __UpperCamelCase						,       __UpperCamelCase						,       __UpperCamelCase						,       overwrite_labels=__UpperCamelCase						,       overwrite_examples=__UpperCamelCase  )




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase=None						,       UpperCAmelCase=None						,       UpperCAmelCase=False						,       UpperCAmelCase=False  )     ->       Dict:






								'''simple docstring'''






								if labels is not None and len(__UpperCamelCase  ) != len(__UpperCamelCase  ):
									raise ValueError(
									    F'Text and labels have mismatched lengths {len(__UpperCamelCase  )} and {len(__UpperCamelCase  )}'  )
								if ids is not None and len(__UpperCamelCase  ) != len(__UpperCamelCase  ):
									raise ValueError(F'Text and ids have mismatched lengths {len(__UpperCamelCase  )} and {len(__UpperCamelCase  )}'  )
								if ids is None:
									lowercase_			 = [None] * len(__UpperCamelCase  )
								if labels is None:
									lowercase_			 = [None] * len(__UpperCamelCase  )
								lowercase_			 = []
								lowercase_			 = set()
								for text_or_text_and_label, label, guid in zip(__UpperCamelCase						,       __UpperCamelCase						,       __UpperCamelCase  ):
									if isinstance(__UpperCamelCase						,       (tuple, list)  ) and label is None:
										lowercase_ ,	lowercase_			 = text_or_text_and_label
									else:
										lowercase_			 = text_or_text_and_label
									added_labels.add(__UpperCamelCase  )
									examples.append(InputExample(guid=__UpperCamelCase						,       text_a=__UpperCamelCase						,       text_b=__UpperCamelCase						,       label=__UpperCamelCase  )  )

								# Update examples
								if overwrite_examples:
									lowercase_			 = examples
								else:
									self.examples.extend(__UpperCamelCase  )

								# Update labels
								if overwrite_labels:
									lowercase_			 = list(__UpperCamelCase  )
								else:
									lowercase_			 = list(set(self.labels  ).union(__UpperCamelCase  )  )

								return self.examples




							def 				A__      (  self						,       UpperCAmelCase						,       UpperCAmelCase=None						,       UpperCAmelCase=False						,       UpperCAmelCase=0						,       UpperCAmelCase=True						,       UpperCAmelCase=None						,       )     ->       str:






								'''simple docstring'''






								if max_length is None:
									lowercase_			 = tokenizer.max_len

								lowercase_			 = {label: i for i, label in enumerate(self.labels  )}

								lowercase_			 = []
								for ex_index, example in enumerate(self.examples  ):
									if ex_index % 10000 == 0:
										logger.info(F'Tokenizing example {ex_index}'  )

									lowercase_			 = tokenizer.encode(
									    example.text_a						,       add_special_tokens=__UpperCamelCase						,       max_length=min(__UpperCamelCase						,       tokenizer.max_len  )						,       )
									all_input_ids.append(__UpperCamelCase  )

								lowercase_			 = max(len(__UpperCamelCase  ) for input_ids in all_input_ids  )

								lowercase_			 = []
								for ex_index, (input_ids, example) in enumerate(zip(__UpperCamelCase						,       self.examples  )  ):
									if ex_index % 10000 == 0:
										logger.info(F'Writing example {ex_index}/{len(self.examples  )}'  )
									# The mask has 1 for real tokens and 0 for padding tokens. Only real
									# tokens are attended to.
									lowercase_			 = [1 if mask_padding_with_zero else 0] * len(__UpperCamelCase  )

									# Zero-pad up to the sequence length.
									lowercase_			 = batch_length - len(__UpperCamelCase  )
									if pad_on_left:
										lowercase_			 = ([pad_token] * padding_length) + input_ids
										lowercase_			 = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
									else:
										lowercase_			 = input_ids + ([pad_token] * padding_length)
										lowercase_			 = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

									if len(__UpperCamelCase  ) != batch_length:
										raise ValueError(F'Error with input length {len(__UpperCamelCase  )} vs {batch_length}'  )
									if len(__UpperCamelCase  ) != batch_length:
										raise ValueError(F'Error with input length {len(__UpperCamelCase  )} vs {batch_length}'  )

									if self.mode == "classification":
										lowercase_			 = label_map[example.label]
									elif self.mode == "regression":
										lowercase_			 = float(example.label  )
									else:
										raise ValueError(self.mode  )

									if ex_index < 5 and self.verbose:
										logger.info("*** Example ***"  )
										logger.info(F'guid: {example.guid}'  )
										logger.info(F'input_ids: {" ".join([str(__UpperCamelCase  ) for x in input_ids]  )}'  )
										logger.info(F'attention_mask: {" ".join([str(__UpperCamelCase  ) for x in attention_mask]  )}'  )
										logger.info(F'label: {example.label} (id = {label})'  )

									features.append(InputFeatures(input_ids=__UpperCamelCase						,       attention_mask=__UpperCamelCase						,       label=__UpperCamelCase  )  )

								if return_tensors is None:
									return features
								elif return_tensors == "tf":
									if not is_tf_available():
										raise RuntimeError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported"  )
									import tensorflow as tf

									def gen():
										for ex in features:
											yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

									lowercase_			 = tf.data.Dataset.from_generator(
									    __UpperCamelCase						,       ({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa)						,       ({"input_ids": tf.TensorShape([None]  ), "attention_mask": tf.TensorShape([None]  )}, tf.TensorShape([]  ))						,       )
									return dataset
								elif return_tensors == "pt":
									if not is_torch_available():
										raise RuntimeError("return_tensors set to 'pt' but PyTorch can't be imported"  )
									import torch
									from torch.utils.data import TensorDataset

									lowercase_			 = torch.tensor([f.input_ids for f in features]						,       dtype=torch.long  )
									lowercase_			 = torch.tensor([f.attention_mask for f in features]						,       dtype=torch.long  )
									if self.mode == "classification":
										lowercase_			 = torch.tensor([f.label for f in features]						,       dtype=torch.long  )
									elif self.mode == "regression":
										lowercase_			 = torch.tensor([f.label for f in features]						,       dtype=torch.float  )

									lowercase_			 = TensorDataset(__UpperCamelCase						,       __UpperCamelCase						,       __UpperCamelCase  )
									return dataset
								else:
									raise ValueError("return_tensors should be one of 'tf' or 'pt'"  )






'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      foo







   def __call__(	self			):
       return self.foo

class    lowerCamelCase_				(						datasets.Dataset	):






   '''simple docstring'''

   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      callable


   def __call__(	self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       if self.callable is not None:
           return self.callable(*__UpperCamelCase  ,    **__UpperCamelCase			)







   def __getstate__(	self			):
       raise pickle.PicklingError()

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       def encode(__UpperCamelCase			):
           return tokenizer(__UpperCamelCase			)

       # TODO: add hash consistency tests across sessions
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-cased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       tokenizer("Hello world !"			)  # call once to change the tokenizer's cache
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_regex
   def 						_UpperCamelCase							(	self			):
       import regex

       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("bar"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       def func():
           return foo

       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()

       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      (
           "co_argcount",
           "co_kwonlyargcount",
           "co_nlocals",
           "co_stacksize",
           "co_flags",
           "co_code",
           "co_consts",
           "co_names",
           "co_varnames",
           "co_filename",
           "co_name",
           "co_firstlineno",
           "co_lnotab",
           "co_freevars",
           "co_cellvars",
       )

       def _create_code_pyaa(*__UpperCamelCase			):
           if hasattr(args[-3]  ,    "encode"			):
               UpperCamelCase__	     =      list(__UpperCamelCase			)
               UpperCamelCase__	     =      args[-3].encode()  # co_lnotab
               UpperCamelCase__	     =      args[-10].encode()  # co_code
           if hasattr(__UpperCamelCase  ,    "co_posonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(*__UpperCamelCase			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(args[0]  ,    0  ,    *args[1:]			)
               return CodeType(args[0]  ,    0  ,    0  ,    *args[1:]			)
           elif hasattr(__UpperCamelCase  ,    "co_kwonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(args[0]  ,    *args[2:]			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(*__UpperCamelCase			)
               return CodeType(args[0]  ,    0  ,    *args[1:]			)
           if len(__UpperCamelCase			) == 16:
               return CodeType(args[0]  ,    *args[3:]			)
           elif len(__UpperCamelCase			) == 15:
               return CodeType(args[0]  ,    *args[2:]			)
           return CodeType(*__UpperCamelCase			)

       def create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			):
           def func():
               return returned_obj

           UpperCamelCase__	     =      func.__code__
           # Use _create_code from dill in order to make it work for different python versions
           if datasets.config.PY_VERSION < version.parse("3.8"			):
               UpperCamelCase__	     =      _create_code_pyaa(
                   *[getattr(__UpperCamelCase  ,    __UpperCamelCase			) if k != "co_filename" else co_filename for k in code_args_pyaa]			)
           else:
               UpperCamelCase__	     =      code.replace(co_filename=__UpperCamelCase			)
           return FunctionType(__UpperCamelCase  ,    func.__globals__  ,    func.__name__  ,    func.__defaults__  ,    func.__closure__			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-5-713f6613acf3>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "654123987.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      [0], [1]

       def func():
           return foo, bar

       UpperCamelCase__	     =      "__main__"

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"foo": foo, "bar": bar}

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"bar": bar, "foo": foo}

       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("there"			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("there"			)			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["there", Foo("there"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       with self.assertRaises(pickle.PicklingError			):
           Hasher.hash(UnpicklableCallable(Foo("hello"			)			)			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "abc"
       UpperCamelCase__	     =      [string, string]  # two strings have the same ids
       UpperCamelCase__	     =      [string, string]
       UpperCamelCase__	     =      json.loads(f'["{string}", "{string}"]'			)  # two strings have different ids
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIsNot(obja[0]  ,    __UpperCamelCase			)
       self.assertIsNot(obja[0]  ,    obja[1]			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_tiktoken
   def 						_UpperCamelCase							(	self			):
       import tiktoken

       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("r50k_base"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_torch
   def 						_UpperCamelCase							(	self			):
       import torch

       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([2.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_spacy
   @require_spacy_model("en_core_web_sm"			)
   @require_spacy_model("fr_core_news_sm"			)
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       import spacy

       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("fr_core_news_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)





@pytest.mark.integration
def     snake_case__			(lowerCAmelCase_						):
    UpperCamelCase__	     =      tmp_path / "dir1"
    UpperCamelCase__	     =      tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    UpperCamelCase__	     =      "script.py"
    UpperCamelCase__	     =      dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
    UpperCamelCase__	     =      dira / script_filename
    UpperCamelCase__	     =      dira / script_filename
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    assert fingerprinta == fingerprinta





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}
    UpperCamelCase__	     =      DatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						)
    UpperCamelCase__	     =      dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2						) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="1.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="2.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3





def     snake_case__			():
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    UpperCamelCase__	     =      Hasher()
    hasher.update(lambda lowerCAmelCase_						: x						)




