



'''simple docstring'''







import unittest
from unittest.util import safe_repr

from transformers import AutoTokenizer, RwkvConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          RWKV_PRETRAINED_MODEL_ARCHIVE_LIST,
          RwkvForCausalLM,
          RwkvModel,
      )
      from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_0
else:
      lowercase__  :	List[Any]            =      False

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=14  ,    __UpperCamelCase=7  ,    __UpperCamelCase=True  ,    __UpperCamelCase=False  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=True  ,    __UpperCamelCase=99  ,    __UpperCamelCase=32  ,    __UpperCamelCase=5  ,    __UpperCamelCase=37  ,    __UpperCamelCase="gelu"  ,    __UpperCamelCase=0.1  ,    __UpperCamelCase=0.1  ,    __UpperCamelCase=5_12  ,    __UpperCamelCase=16  ,    __UpperCamelCase=2  ,    __UpperCamelCase=3  ,    __UpperCamelCase=4  ,    __UpperCamelCase=None  ,    ):
       UpperCamelCase__	     =      parent
       UpperCamelCase__	     =      batch_size
       UpperCamelCase__	     =      seq_length
       UpperCamelCase__	     =      is_training
       UpperCamelCase__	     =      use_token_type_ids
       UpperCamelCase__	     =      use_input_mask
       UpperCamelCase__	     =      use_labels
       UpperCamelCase__	     =      use_mc_token_ids
       UpperCamelCase__	     =      vocab_size
       UpperCamelCase__	     =      hidden_size
       UpperCamelCase__	     =      num_hidden_layers
       UpperCamelCase__	     =      intermediate_size
       UpperCamelCase__	     =      hidden_act
       UpperCamelCase__	     =      hidden_dropout_prob
       UpperCamelCase__	     =      attention_probs_dropout_prob
       UpperCamelCase__	     =      max_position_embeddings
       UpperCamelCase__	     =      type_vocab_size
       UpperCamelCase__	     =      type_sequence_label_size
       UpperCamelCase__	     =      num_labels
       UpperCamelCase__	     =      num_choices
       UpperCamelCase__	     =      scope
       UpperCamelCase__	     =      vocab_size - 1
       UpperCamelCase__	     =      vocab_size - 1
       UpperCamelCase__	     =      vocab_size - 1


   def 						_UpperCamelCase							(	self			):
       return RwkvConfig.from_pretrained("sgugger/rwkv-4-pile-7b"			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False			):
       UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size			)

       UpperCamelCase__	     =      None
       if self.use_input_mask:
           UpperCamelCase__	     =      random_attention_mask([self.batch_size, self.seq_length]			)

       UpperCamelCase__	     =      None
       if self.use_token_type_ids:
           UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    self.type_vocab_size			)

       UpperCamelCase__	     =      None
       if self.use_mc_token_ids:
           UpperCamelCase__	     =      ids_tensor([self.batch_size, self.num_choices]  ,    self.seq_length			)

       UpperCamelCase__	     =      None
       UpperCamelCase__	     =      None
       UpperCamelCase__	     =      None
       if self.use_labels:
           UpperCamelCase__	     =      ids_tensor([self.batch_size]  ,    self.type_sequence_label_size			)
           UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    self.num_labels			)
           UpperCamelCase__	     =      ids_tensor([self.batch_size]  ,    self.num_choices			)

       UpperCamelCase__	     =      self.get_config(
           gradient_checkpointing=__UpperCamelCase  ,    scale_attn_by_inverse_layer_idx=__UpperCamelCase  ,    reorder_and_upcast_attn=__UpperCamelCase  ,    )

       return (
           config,
           input_ids,
           input_mask,
           None,
           token_type_ids,
           mc_token_ids,
           sequence_labels,
           token_labels,
           choice_labels,
       )


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False			):
       return RwkvConfig(
           vocab_size=self.vocab_size  ,    hidden_size=self.hidden_size  ,    num_hidden_layers=self.num_hidden_layers  ,    intermediate_size=self.intermediate_size  ,    activation_function=self.hidden_act  ,    resid_pdrop=self.hidden_dropout_prob  ,    attn_pdrop=self.attention_probs_dropout_prob  ,    n_positions=self.max_position_embeddings  ,    type_vocab_size=self.type_vocab_size  ,    use_cache=__UpperCamelCase  ,    bos_token_id=self.bos_token_id  ,    eos_token_id=self.eos_token_id  ,    pad_token_id=self.pad_token_id  ,    gradient_checkpointing=__UpperCamelCase  ,    scale_attn_by_inverse_layer_idx=__UpperCamelCase  ,    reorder_and_upcast_attn=__UpperCamelCase  ,    )


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.get_config()
       UpperCamelCase__	     =      3_00
       return config


   def 						_UpperCamelCase							(	self			):
       (
           (
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	
       )	     =      self.prepare_config_and_inputs()

       UpperCamelCase__	     =      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
       UpperCamelCase__	     =      ids_tensor([self.batch_size, self.seq_length]  ,    vocab_size=2			)

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
           token_type_ids,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    *__UpperCamelCase			):
       UpperCamelCase__	     =      True
       UpperCamelCase__	     =      RwkvModel(config=__UpperCamelCase			)
       model.to(__UpperCamelCase			)
       model.eval()

       UpperCamelCase__	     =      model(__UpperCamelCase			)

       self.parent.assertEqual(result.last_hidden_state.shape  ,    (self.batch_size, self.seq_length, self.hidden_size)			)
       self.parent.assertEqual(len(result.hidden_states			)  ,    config.num_hidden_layers + 1			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    *__UpperCamelCase			):
       UpperCamelCase__	     =      RwkvForCausalLM(__UpperCamelCase			)
       model.to(__UpperCamelCase			)
       model.eval()

       UpperCamelCase__	     =      model(__UpperCamelCase  ,    labels=__UpperCamelCase			)
       self.parent.assertEqual(result.loss.shape  ,    ()			)
       self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    *__UpperCamelCase			):
       UpperCamelCase__	     =      RwkvModel(config=__UpperCamelCase			)
       model.to(__UpperCamelCase			)
       model.eval()

       UpperCamelCase__	     =      model(__UpperCamelCase			)
       UpperCamelCase__	     =      outputs.last_hidden_state

       UpperCamelCase__	     =      model(input_ids[:, :2]			)
       UpperCamelCase__	     =      outputs.last_hidden_state

       # Using the state computed on the first inputs, we will get the same output
       UpperCamelCase__	     =      model(input_ids[:, 2:]  ,    state=outputs.state			)
       UpperCamelCase__	     =      outputs.last_hidden_state

       self.parent.assertTrue(torch.allclose(torch.cat([output_one, output_two]  ,    dim=1			)  ,    __UpperCamelCase  ,    atol=1e-5			)			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    __UpperCamelCase=False			):
       UpperCamelCase__	     =      RwkvForCausalLM(__UpperCamelCase			)
       model.to(__UpperCamelCase			)
       if gradient_checkpointing:
           model.gradient_checkpointing_enable()

       UpperCamelCase__	     =      model(__UpperCamelCase  ,    labels=__UpperCamelCase			)
       self.parent.assertEqual(result.loss.shape  ,    ()			)
       self.parent.assertEqual(result.logits.shape  ,    (self.batch_size, self.seq_length, self.vocab_size)			)
       result.loss.backward()







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.prepare_config_and_inputs()

       (
           (
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	(
           UpperCamelCase__
       )     ,	
       )	     =      config_and_inputs

       UpperCamelCase__	     =      {"input_ids": input_ids}

       return config, inputs_dict

@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0       ,     reason="See https://github.com/huggingface/transformers/pull/24204"	)
@require_torch
class    lowerCamelCase_				(						lowerCAmelCase_       ,     lowerCAmelCase_       ,     lowerCAmelCase_       ,     unittest.TestCase	):






   '''simple docstring'''

   A			:  Union[str, Any]					=						(RwkvModel, RwkvForCausalLM) if is_torch_available() else ()
   A			:  Dict					=						(
       {"feature-extraction": RwkvModel, "text-generation": RwkvForCausalLM} if is_torch_available() else {}
   )
   # all_generative_model_classes = (RwkvForCausalLM,) if is_torch_available() else ()
   A			:  List[Any]					=						False
   A			:  Any					=						False
   A			:  Dict					=						False
   A			:  Optional[int]					=						False
   A			:  Tuple					=						False  # Rwkv does not support head masking


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      RwkvModelTester(self			)
       UpperCamelCase__	     =      ConfigTester(
           self  ,    config_class=__UpperCamelCase  ,    n_embd=37  ,    common_properties=["hidden_size", "num_hidden_layers"]			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       if isinstance(__UpperCamelCase  ,    torch.Tensor			):
           UpperCamelCase__     ,	UpperCamelCase__	     =      member.max().item(), member.min().item()
       elif isinstance(__UpperCamelCase  ,    __UpperCamelCase			) or isinstance(__UpperCamelCase  ,    __UpperCamelCase			):
           UpperCamelCase__     ,	UpperCamelCase__	     =      max(__UpperCamelCase			), min(__UpperCamelCase			)

       if not isinstance(__UpperCamelCase  ,    __UpperCamelCase			):
           raise TypeError("container should be a list or tuple"			)
       elif len(__UpperCamelCase			) != 2:
           raise ValueError("container should have 2 elements"			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      container

       UpperCamelCase__	     =      (min_value >= expected_min) and (max_value <= expected_max)

       if not is_inside_interval:
           UpperCamelCase__	     =      "%s not found in %s" % (safe_repr(__UpperCamelCase			), safe_repr(__UpperCamelCase			))
           self.fail(self._formatMessage(__UpperCamelCase  ,    __UpperCamelCase			)			)


   def 						_UpperCamelCase							(	self			):
       self.config_tester.run_common_tests()


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_rwkv_model(*__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_causl_lm(*__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_state_equivalency(*__UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
           UpperCamelCase__	     =      model_class(config=__UpperCamelCase			)
           for name, param in model.named_parameters():
               if "time_decay" in name:
                   if param.requires_grad:
                       self.assertTrue(param.data.max().item() == 3.0			)
                       self.assertTrue(param.data.min().item() == -5.0			)
               elif "time_first" in name:
                   if param.requires_grad:
                       # check if it's a ones like
                       self.assertTrue(torch.allclose(param.data  ,    torch.ones_like(param.data			)  ,    atol=1e-5  ,    rtol=1e-5			)			)
               elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]			):
                   if param.requires_grad:
                       self.assertInterval(
                           param.data  ,    [0.0, 1.0]  ,    msg=f'Parameter {name} of model {model_class} seems not properly initialized'  ,    )
               elif "time_mix_value" in name:
                   if param.requires_grad:
                       self.assertInterval(
                           param.data  ,    [0.0, 1.3]  ,    msg=f'Parameter {name} of model {model_class} seems not properly initialized'  ,    )


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__	     =      True

       UpperCamelCase__	     =      getattr(self.model_tester  ,    "seq_length"  ,    __UpperCamelCase			)

       for model_class in self.all_model_classes:
           UpperCamelCase__	     =      True
           UpperCamelCase__	     =      False
           UpperCamelCase__	     =      True
           UpperCamelCase__	     =      model_class(__UpperCamelCase			)
           model.to(__UpperCamelCase			)
           model.eval()

           UpperCamelCase__	     =      self._prepare_for_class(__UpperCamelCase  ,    __UpperCamelCase			)
           UpperCamelCase__	     =      inputs["input_ids"].shape[0]
           with torch.no_grad():
               UpperCamelCase__	     =      model(**__UpperCamelCase			)
           UpperCamelCase__	     =      outputs.attentions
           self.assertEqual(len(__UpperCamelCase			)  ,    self.model_tester.num_hidden_layers			)

           # check that output_attentions also work using config
           del inputs_dict["output_attentions"]
           UpperCamelCase__	     =      True
           UpperCamelCase__	     =      model_class(__UpperCamelCase			)
           model.to(__UpperCamelCase			)
           model.eval()

           UpperCamelCase__	     =      self._prepare_for_class(__UpperCamelCase  ,    __UpperCamelCase			)
           UpperCamelCase__	     =      inputs["input_ids"].shape[0]
           with torch.no_grad():
               UpperCamelCase__	     =      model(**__UpperCamelCase			)
           UpperCamelCase__	     =      outputs.attentions
           self.assertEqual(len(__UpperCamelCase			)  ,    self.model_tester.num_hidden_layers			)

           self.assertListEqual(
               list(attentions[0].shape[-3:]			)  ,    [batch_size, seq_len, config.hidden_size]  ,    )
           UpperCamelCase__	     =      len(__UpperCamelCase			)

           # Check attention is always last and order is fine
           UpperCamelCase__	     =      True
           UpperCamelCase__	     =      True
           UpperCamelCase__	     =      model_class(__UpperCamelCase			)
           model.to(__UpperCamelCase			)
           model.eval()

           UpperCamelCase__	     =      self._prepare_for_class(__UpperCamelCase  ,    __UpperCamelCase			)
           UpperCamelCase__	     =      inputs["input_ids"].shape[0]
           with torch.no_grad():
               UpperCamelCase__	     =      model(**__UpperCamelCase			)

           UpperCamelCase__	     =      1
           self.assertEqual(out_len + added_hidden_states  ,    len(__UpperCamelCase			)			)

           UpperCamelCase__	     =      outputs.attentions

           self.assertEqual(len(__UpperCamelCase			)  ,    self.model_tester.num_hidden_layers			)
           self.assertListEqual(
               list(self_attentions[0].shape[-3:]			)  ,    [batch_size, seq_len, config.hidden_size]  ,    )







   @slow
   def 						_UpperCamelCase							(	self			):
       for model_name in RWKV_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           UpperCamelCase__	     =      RwkvModel.from_pretrained(__UpperCamelCase			)
           self.assertIsNotNone(__UpperCamelCase			)

@unittest.skipIf(
    not is_torch_greater_or_equal_than_2_0       ,     reason="See https://github.com/huggingface/transformers/pull/24204"	)
@slow
class    lowerCamelCase_				(						unittest.TestCase	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "RWKV/rwkv-4-169m-pile"
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained(self.model_id			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "Hello my name is Jasmine and I am a newbie to the"
       UpperCamelCase__	     =      RwkvForCausalLM.from_pretrained(self.model_id			).to(__UpperCamelCase			)

       UpperCamelCase__	     =      self.tokenizer("Hello my name is"  ,    return_tensors="pt"			).input_ids.to(__UpperCamelCase			)
       UpperCamelCase__	     =      model.generate(__UpperCamelCase  ,    max_new_tokens=10			)
       UpperCamelCase__	     =      self.tokenizer.decode(output[0].tolist()			)

       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "Hello my name is Jasmine and I am a newbie to the"

       UpperCamelCase__	     =      self.tokenizer("Hello my name is"  ,    return_tensors="pt"			).input_ids.to(__UpperCamelCase			)
       UpperCamelCase__	     =      RwkvForCausalLM.from_pretrained(self.model_id  ,    torch_dtype=torch.bfloataa			).to(__UpperCamelCase			)

       UpperCamelCase__	     =      model.generate(__UpperCamelCase  ,    max_new_tokens=10			)
       UpperCamelCase__	     =      self.tokenizer.decode(output[0].tolist()			)

       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)









'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      foo







   def __call__(	self			):
       return self.foo

class    lowerCamelCase_				(						datasets.Dataset	):






   '''simple docstring'''

   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      callable


   def __call__(	self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       if self.callable is not None:
           return self.callable(*__UpperCamelCase  ,    **__UpperCamelCase			)







   def __getstate__(	self			):
       raise pickle.PicklingError()

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       def encode(__UpperCamelCase			):
           return tokenizer(__UpperCamelCase			)

       # TODO: add hash consistency tests across sessions
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-cased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       tokenizer("Hello world !"			)  # call once to change the tokenizer's cache
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_regex
   def 						_UpperCamelCase							(	self			):
       import regex

       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("bar"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       def func():
           return foo

       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()

       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      (
           "co_argcount",
           "co_kwonlyargcount",
           "co_nlocals",
           "co_stacksize",
           "co_flags",
           "co_code",
           "co_consts",
           "co_names",
           "co_varnames",
           "co_filename",
           "co_name",
           "co_firstlineno",
           "co_lnotab",
           "co_freevars",
           "co_cellvars",
       )

       def _create_code_pyaa(*__UpperCamelCase			):
           if hasattr(args[-3]  ,    "encode"			):
               UpperCamelCase__	     =      list(__UpperCamelCase			)
               UpperCamelCase__	     =      args[-3].encode()  # co_lnotab
               UpperCamelCase__	     =      args[-10].encode()  # co_code
           if hasattr(__UpperCamelCase  ,    "co_posonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(*__UpperCamelCase			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(args[0]  ,    0  ,    *args[1:]			)
               return CodeType(args[0]  ,    0  ,    0  ,    *args[1:]			)
           elif hasattr(__UpperCamelCase  ,    "co_kwonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(args[0]  ,    *args[2:]			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(*__UpperCamelCase			)
               return CodeType(args[0]  ,    0  ,    *args[1:]			)
           if len(__UpperCamelCase			) == 16:
               return CodeType(args[0]  ,    *args[3:]			)
           elif len(__UpperCamelCase			) == 15:
               return CodeType(args[0]  ,    *args[2:]			)
           return CodeType(*__UpperCamelCase			)

       def create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			):
           def func():
               return returned_obj

           UpperCamelCase__	     =      func.__code__
           # Use _create_code from dill in order to make it work for different python versions
           if datasets.config.PY_VERSION < version.parse("3.8"			):
               UpperCamelCase__	     =      _create_code_pyaa(
                   *[getattr(__UpperCamelCase  ,    __UpperCamelCase			) if k != "co_filename" else co_filename for k in code_args_pyaa]			)
           else:
               UpperCamelCase__	     =      code.replace(co_filename=__UpperCamelCase			)
           return FunctionType(__UpperCamelCase  ,    func.__globals__  ,    func.__name__  ,    func.__defaults__  ,    func.__closure__			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-5-713f6613acf3>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "654123987.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      [0], [1]

       def func():
           return foo, bar

       UpperCamelCase__	     =      "__main__"

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"foo": foo, "bar": bar}

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"bar": bar, "foo": foo}

       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("there"			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("there"			)			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["there", Foo("there"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       with self.assertRaises(pickle.PicklingError			):
           Hasher.hash(UnpicklableCallable(Foo("hello"			)			)			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "abc"
       UpperCamelCase__	     =      [string, string]  # two strings have the same ids
       UpperCamelCase__	     =      [string, string]
       UpperCamelCase__	     =      json.loads(f'["{string}", "{string}"]'			)  # two strings have different ids
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIsNot(obja[0]  ,    __UpperCamelCase			)
       self.assertIsNot(obja[0]  ,    obja[1]			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_tiktoken
   def 						_UpperCamelCase							(	self			):
       import tiktoken

       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("r50k_base"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_torch
   def 						_UpperCamelCase							(	self			):
       import torch

       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([2.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_spacy
   @require_spacy_model("en_core_web_sm"			)
   @require_spacy_model("fr_core_news_sm"			)
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       import spacy

       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("fr_core_news_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)





@pytest.mark.integration
def     snake_case__			(lowerCAmelCase_						):
    UpperCamelCase__	     =      tmp_path / "dir1"
    UpperCamelCase__	     =      tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    UpperCamelCase__	     =      "script.py"
    UpperCamelCase__	     =      dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
    UpperCamelCase__	     =      dira / script_filename
    UpperCamelCase__	     =      dira / script_filename
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    assert fingerprinta == fingerprinta





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}
    UpperCamelCase__	     =      DatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						)
    UpperCamelCase__	     =      dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2						) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="1.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="2.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3





def     snake_case__			():
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    UpperCamelCase__	     =      Hasher()
    hasher.update(lambda lowerCAmelCase_						: x						)




