




import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import MBartConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
	import os

	# The slow tests are often failing with OOM error on GPU
	# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
	# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
	__A						:		Optional[int]          =						"platform"

	import jax
	import jax.numpy as jnp

	from transformers import AutoTokenizer
	from transformers.models.mbart.modeling_flax_mbart import (
	    FlaxMBartForConditionalGeneration,
	    FlaxMBartForQuestionAnswering,
	    FlaxMBartForSequenceClassification,
	    FlaxMBartModel,
	    shift_tokens_right,
	)





def      UpperCAmelCase  (						UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			)     ->     str:







				'''simple docstring'''


				if attention_mask is None:
								__lowerCAmelCase					=  np.where(input_ids != config.pad_token_id						,			1						,			0   )
				if decoder_attention_mask is None:
								__lowerCAmelCase					=  np.where(decoder_input_ids != config.pad_token_id						,			1						,			0   )
				if head_mask is None:
								__lowerCAmelCase					=  np.ones((config.encoder_layers, config.encoder_attention_heads)   )
				if decoder_head_mask is None:
								__lowerCAmelCase					=  np.ones((config.decoder_layers, config.decoder_attention_heads)   )
				if cross_attn_head_mask is None:
								__lowerCAmelCase					=  np.ones((config.decoder_layers, config.decoder_attention_heads)   )
				return {
				    "input_ids": input_ids,
				    "decoder_input_ids": decoder_input_ids,
				    "attention_mask": attention_mask,
				    "decoder_attention_mask": decoder_attention_mask,
				}

class 		lowercase_       (	unittest.TestCase				):




					def __init__(   self:		Dict,     _lowercase:		List[Any],     _lowercase:		Any=13,     _lowercase:		int=7,     _lowercase:		str=True,     _lowercase:		List[str]=False,     _lowercase:		Any=99,     _lowercase:		str=16,     _lowercase:		Optional[int]=2,     _lowercase:		List[str]=4,     _lowercase:		List[str]=4,     _lowercase:		Optional[int]="gelu",     _lowercase:		Any=0.1,     _lowercase:		Union[str, Any]=0.1,     _lowercase:		Dict=32,     _lowercase:		int=2,     _lowercase:		Union[str, Any]=1,     _lowercase:		Any=0,     _lowercase:		int=2,     _lowercase:		List[Any]=0.02,     ):


									'''simple docstring'''


									__lowerCAmelCase					=  parent
									__lowerCAmelCase					=  batch_size
									__lowerCAmelCase					=  seq_length
									__lowerCAmelCase					=  is_training
									__lowerCAmelCase					=  use_labels
									__lowerCAmelCase					=  vocab_size
									__lowerCAmelCase					=  hidden_size
									__lowerCAmelCase					=  num_hidden_layers
									__lowerCAmelCase					=  num_attention_heads
									__lowerCAmelCase					=  intermediate_size
									__lowerCAmelCase					=  hidden_act
									__lowerCAmelCase					=  hidden_dropout_prob
									__lowerCAmelCase					=  attention_probs_dropout_prob
									__lowerCAmelCase					=  max_position_embeddings
									__lowerCAmelCase					=  eos_token_id
									__lowerCAmelCase					=  pad_token_id
									__lowerCAmelCase					=  bos_token_id
									__lowerCAmelCase					=  decoder_start_token_id
									__lowerCAmelCase					=  initializer_range




					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									__lowerCAmelCase					=  np.clip(ids_tensor([self.batch_size, self.seq_length - 1],     self.vocab_size),     3,     self.vocab_size)
									__lowerCAmelCase					=  np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1),     dtype=np.intaa)),     -1)

									__lowerCAmelCase					=  shift_tokens_right(_lowercase,     1)

									__lowerCAmelCase					=  MBartConfig(
									    vocab_size=self.vocab_size,     d_model=self.hidden_size,     encoder_layers=self.num_hidden_layers,     decoder_layers=self.num_hidden_layers,     encoder_attention_heads=self.num_attention_heads,     decoder_attention_heads=self.num_attention_heads,     encoder_ffn_dim=self.intermediate_size,     decoder_ffn_dim=self.intermediate_size,     dropout=self.hidden_dropout_prob,     attention_dropout=self.attention_probs_dropout_prob,     max_position_embeddings=self.max_position_embeddings,     eos_token_id=self.eos_token_id,     bos_token_id=self.bos_token_id,     pad_token_id=self.pad_token_id,     decoder_start_token_id=self.decoder_start_token_id,     initializer_range=self.initializer_range,     use_cache=_lowercase,     )
									__lowerCAmelCase					=  prepare_mbart_inputs_dict(_lowercase,     _lowercase,     _lowercase)
									return config, inputs_dict




					def     _lowercase						(   self:		Any):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase					=  self.prepare_config_and_inputs()
									return config, inputs_dict




					def     _lowercase						(   self:		Union[str, Any],     _lowercase:		Union[str, Any],     _lowercase:		Dict,     _lowercase:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  20
									__lowerCAmelCase					=  model_class_name(_lowercase)

									__lowerCAmelCase					=  model.encode(inputs_dict["""input_ids"""])

									__lowerCAmelCase	,				__lowerCAmelCase					=  (
									    inputs_dict["""decoder_input_ids"""],
									    inputs_dict["""decoder_attention_mask"""],
									)

									__lowerCAmelCase					=  model.init_cache(decoder_input_ids.shape[0],     _lowercase,     _lowercase)
									__lowerCAmelCase					=  jnp.ones((decoder_input_ids.shape[0], max_decoder_length),     dtype="""i4""")

									__lowerCAmelCase					=  jnp.broadcast_to(
									    jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :],     (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1),     )
									__lowerCAmelCase					=  model.decode(
									    decoder_input_ids[:, :-1],     _lowercase,     decoder_attention_mask=_lowercase,     past_key_values=_lowercase,     decoder_position_ids=_lowercase,     )

									__lowerCAmelCase					=  jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]],     dtype="""i4""")
									__lowerCAmelCase					=  model.decode(
									    decoder_input_ids[:, -1:],     _lowercase,     decoder_attention_mask=_lowercase,     past_key_values=outputs_cache.past_key_values,     decoder_position_ids=_lowercase,     )

									__lowerCAmelCase					=  model.decode(_lowercase,     _lowercase)

									__lowerCAmelCase					=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])))
									self.parent.assertTrue(diff < 1e-3,     msg=f'''Max diff is {diff}''')







					def     _lowercase						(   self:		List[str],     _lowercase:		int,     _lowercase:		str,     _lowercase:		str):


									'''simple docstring'''


									__lowerCAmelCase					=  20
									__lowerCAmelCase					=  model_class_name(_lowercase)

									__lowerCAmelCase					=  model.encode(inputs_dict["""input_ids"""])

									__lowerCAmelCase	,				__lowerCAmelCase					=  (
									    inputs_dict["""decoder_input_ids"""],
									    inputs_dict["""decoder_attention_mask"""],
									)

									__lowerCAmelCase					=  jnp.concatenate(
									    [
									        decoder_attention_mask,
									        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])),
									    ],     axis=-1,     )

									__lowerCAmelCase					=  model.init_cache(decoder_input_ids.shape[0],     _lowercase,     _lowercase)
									__lowerCAmelCase					=  jnp.broadcast_to(
									    jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :],     (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1),     )

									__lowerCAmelCase					=  model.decode(
									    decoder_input_ids[:, :-1],     _lowercase,     decoder_attention_mask=_lowercase,     past_key_values=_lowercase,     decoder_position_ids=_lowercase,     )
									__lowerCAmelCase					=  jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]],     dtype="""i4""")
									__lowerCAmelCase					=  model.decode(
									    decoder_input_ids[:, -1:],     _lowercase,     past_key_values=outputs_cache.past_key_values,     decoder_attention_mask=_lowercase,     decoder_position_ids=_lowercase,     )

									__lowerCAmelCase					=  model.decode(_lowercase,     _lowercase,     decoder_attention_mask=_lowercase)

									__lowerCAmelCase					=  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])))
									self.parent.assertTrue(diff < 1e-3,     msg=f'''Max diff is {diff}''')





@require_flax
class 		lowercase_       (	unittest.TestCase				):
					__UpperCamelCase     	= 99




					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									__lowerCAmelCase					=  np.array(
									    [
									        [71, 82, 18, 33, 46, 91, 2],
									        [68, 34, 26, 58, 30, 82, 2],
									        [5, 97, 17, 39, 94, 40, 2],
									        [76, 83, 94, 25, 70, 78, 2],
									        [87, 59, 41, 35, 48, 66, 2],
									        [55, 13, 16, 58, 5, 2, 1],  # note padding
									        [64, 27, 31, 51, 12, 75, 2],
									        [52, 64, 86, 17, 83, 39, 2],
									        [48, 61, 9, 24, 71, 82, 2],
									        [26, 1, 60, 48, 22, 13, 2],
									        [21, 5, 62, 28, 14, 76, 2],
									        [45, 98, 37, 86, 59, 48, 2],
									        [70, 70, 50, 9, 28, 0, 2],
									    ],     dtype=np.intaa,     )

									__lowerCAmelCase					=  input_ids.shape[0]
									__lowerCAmelCase					=  MBartConfig(
									    vocab_size=self.vocab_size,     d_model=24,     encoder_layers=2,     decoder_layers=2,     encoder_attention_heads=2,     decoder_attention_heads=2,     encoder_ffn_dim=32,     decoder_ffn_dim=32,     max_position_embeddings=48,     eos_token_id=2,     pad_token_id=1,     bos_token_id=0,     )
									return config, input_ids, batch_size




					def     _lowercase						(   self:		Optional[Any]):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase					=  self._get_config_and_data()
									__lowerCAmelCase					=  FlaxMBartForSequenceClassification(_lowercase)
									__lowerCAmelCase					=  model(input_ids=_lowercase,     decoder_input_ids=_lowercase)
									__lowerCAmelCase					=  (batch_size, config.num_labels)
									self.assertEqual(outputs["""logits"""].shape,     _lowercase)




					def     _lowercase						(   self:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase					=  self._get_config_and_data()
									__lowerCAmelCase					=  FlaxMBartForQuestionAnswering(_lowercase)
									__lowerCAmelCase					=  model(input_ids=_lowercase)

									self.assertEqual(outputs["""start_logits"""].shape,     input_ids.shape)
									self.assertEqual(outputs["""end_logits"""].shape,     input_ids.shape)




					def     _lowercase						(   self:		int):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase					=  self._get_config_and_data()
									__lowerCAmelCase					=  FlaxMBartForConditionalGeneration(_lowercase)
									__lowerCAmelCase					=  lm_model(input_ids=_lowercase)
									__lowerCAmelCase					=  (batch_size, input_ids.shape[1], config.vocab_size)
									self.assertEqual(outputs["""logits"""].shape,     _lowercase)




					def     _lowercase						(   self:		str):


									'''simple docstring'''


									__lowerCAmelCase					=  MBartConfig(
									    vocab_size=self.vocab_size,     d_model=14,     encoder_layers=2,     decoder_layers=2,     encoder_attention_heads=2,     decoder_attention_heads=2,     encoder_ffn_dim=8,     decoder_ffn_dim=8,     max_position_embeddings=48,     )
									__lowerCAmelCase					=  FlaxMBartForConditionalGeneration(_lowercase)
									__lowerCAmelCase					=  np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]],     dtype=np.intaa)
									__lowerCAmelCase					=  np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]],     dtype=np.intaa)
									__lowerCAmelCase					=  lm_model(input_ids=_lowercase,     decoder_input_ids=_lowercase)
									__lowerCAmelCase					=  (*summary.shape, config.vocab_size)
									self.assertEqual(outputs["""logits"""].shape,     _lowercase)







					def     _lowercase						(   self:		Optional[int]):


									'''simple docstring'''


									__lowerCAmelCase					=  np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]],     dtype=np.intaa)
									__lowerCAmelCase					=  shift_tokens_right(_lowercase,     1)
									__lowerCAmelCase					=  np.equal(_lowercase,     1).astype(np.floataa).sum()
									__lowerCAmelCase					=  np.equal(_lowercase,     1).astype(np.floataa).sum()
									self.assertEqual(shifted.shape,     input_ids.shape)
									self.assertEqual(_lowercase,     n_pad_before - 1)
									self.assertTrue(np.equal(shifted[:, 0],     2).all())





@require_flax
class 		lowercase_       (	lowerCAmelCase__						,							unittest.TestCase						,							lowerCAmelCase__				):
					__UpperCamelCase     	= True
					__UpperCamelCase     	= (
					    (
					        FlaxMBartModel,
					        FlaxMBartForConditionalGeneration,
					        FlaxMBartForSequenceClassification,
					        FlaxMBartForQuestionAnswering,
					    )
					    if is_flax_available()
					    else ()
					)
					__UpperCamelCase     	= (FlaxMBartForConditionalGeneration,) if is_flax_available() else ()




					def     _lowercase						(   self:		int):


									'''simple docstring'''


									__lowerCAmelCase					=  FlaxMBartModelTester(self)




					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
													self.model_tester.check_use_cache_forward(_lowercase,     _lowercase,     _lowercase)




					def     _lowercase						(   self:		str):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
													self.model_tester.check_use_cache_forward_with_attn_mask(_lowercase,     _lowercase,     _lowercase)




					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
													with self.subTest(model_class.__name__):
																	__lowerCAmelCase					=  self._prepare_for_class(_lowercase,     _lowercase)
																	__lowerCAmelCase					=  model_class(_lowercase)

																	@jax.jit
																	def encode_jitted(_lowercase:		List[Any],     _lowercase:		Tuple=None,     **_lowercase:		Optional[Any]):
																					return model.encode(input_ids=_lowercase,     attention_mask=_lowercase)

																	with self.subTest("""JIT Enabled"""):
																					__lowerCAmelCase					=  encode_jitted(**_lowercase).to_tuple()

																	with self.subTest("""JIT Disabled"""):
																					with jax.disable_jit():
																									__lowerCAmelCase					=  encode_jitted(**_lowercase).to_tuple()

																	self.assertEqual(len(_lowercase),     len(_lowercase))
																	for jitted_output, output in zip(_lowercase,     _lowercase):
																					self.assertEqual(jitted_output.shape,     output.shape)




					def     _lowercase						(   self:		Any):


									'''simple docstring'''


									__lowerCAmelCase	,				__lowerCAmelCase					=  self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
													with self.subTest(model_class.__name__):
																	__lowerCAmelCase					=  model_class(_lowercase)
																	__lowerCAmelCase					=  model.encode(inputs_dict["""input_ids"""],     inputs_dict["""attention_mask"""])

																	__lowerCAmelCase					=  {
																	    """decoder_input_ids""": inputs_dict["""decoder_input_ids"""],
																	    """decoder_attention_mask""": inputs_dict["""decoder_attention_mask"""],
																	    """encoder_outputs""": encoder_outputs,
																	}

																	@jax.jit
																	def decode_jitted(_lowercase:		Union[str, Any],     _lowercase:		str,     _lowercase:		Optional[Any]):
																					return model.decode(
																					    decoder_input_ids=_lowercase,     decoder_attention_mask=_lowercase,     encoder_outputs=_lowercase,     )

																	with self.subTest("""JIT Enabled"""):
																					__lowerCAmelCase					=  decode_jitted(**_lowercase).to_tuple()

																	with self.subTest("""JIT Disabled"""):
																					with jax.disable_jit():
																									__lowerCAmelCase					=  decode_jitted(**_lowercase).to_tuple()

																	self.assertEqual(len(_lowercase),     len(_lowercase))
																	for jitted_output, output in zip(_lowercase,     _lowercase):
																					self.assertEqual(jitted_output.shape,     output.shape)







					@slow
					def     _lowercase						(   self:		List[str]):


									'''simple docstring'''


									for model_class_name in self.all_model_classes:
													__lowerCAmelCase					=  model_class_name.from_pretrained("""facebook/mbart-large-cc25""",     from_pt=_lowercase)
													# FlaxMBartForSequenceClassification expects eos token in input_ids
													__lowerCAmelCase					=  np.ones((1, 1)) * model.config.eos_token_id
													__lowerCAmelCase					=  model(_lowercase)
													self.assertIsNotNone(_lowercase)





@require_flax
@require_sentencepiece
@require_tokenizers
class 		lowercase_       (	unittest.TestCase				):
					__UpperCamelCase     	= [
					    " UN Chief Says There Is No Military Solution in Syria",
					]
					__UpperCamelCase     	= [
					    "Şeful ONU declară că nu există o soluţie militară în Siria",
					]
					__UpperCamelCase     	= "facebook/mbart-large-en-ro"




					@cached_property
					def     _lowercase						(   self:		Dict):


									'''simple docstring'''


									return AutoTokenizer.from_pretrained(self.model_name)




					@cached_property
					def     _lowercase						(   self:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  FlaxMBartForConditionalGeneration.from_pretrained(self.model_name,     from_pt=_lowercase)
									return model




					def     _lowercase						(   self:		Tuple,     **_lowercase:		Tuple):


									'''simple docstring'''


									__lowerCAmelCase					=  self.translate_src_text(**_lowercase)
									self.assertListEqual(self.expected_text,     _lowercase)




					def     _lowercase						(   self:		Optional[Any],     **_lowercase:		List[Any]):


									'''simple docstring'''


									__lowerCAmelCase					=  self.tokenizer(self.src_text,     **_lowercase,     return_tensors="""np""")
									__lowerCAmelCase					=  self.model.generate(
									    model_inputs.input_ids,     attention_mask=model_inputs.attention_mask,     decoder_start_token_id=self.tokenizer.lang_code_to_id["""ro_RO"""],     early_stopping=_lowercase,     num_beams=2,     ).sequences
									__lowerCAmelCase					=  self.tokenizer.batch_decode(_lowercase,     skip_special_tokens=_lowercase)
									return generated_words







					@slow
					def     _lowercase						(   self:		Optional[int]):


									'''simple docstring'''


									self._assert_generated_batch_equal_expected()










'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      foo







   def __call__(	self			):
       return self.foo

class    lowerCamelCase_				(						datasets.Dataset	):






   '''simple docstring'''

   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)







   @fingerprint_transform(inplace=__UpperCamelCase			)
   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       return DatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

class    lowerCamelCase_				:






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase			):
       UpperCamelCase__	     =      callable


   def __call__(	self  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
       if self.callable is not None:
           return self.callable(*__UpperCamelCase  ,    **__UpperCamelCase			)







   def __getstate__(	self			):
       raise pickle.PicklingError()

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       def encode(__UpperCamelCase			):
           return tokenizer(__UpperCamelCase			)

       # TODO: add hash consistency tests across sessions
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-cased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("bert-base-uncased"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase			: tokenizer(__UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_transformers
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       from transformers import AutoTokenizer

       UpperCamelCase__	     =      AutoTokenizer.from_pretrained("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       tokenizer("Hello world !"			)  # call once to change the tokenizer's cache
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_regex
   def 						_UpperCamelCase							(	self			):
       import regex

       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("bar"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      regex.Regex("foo"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       def func():
           return foo

       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()

       def func():
           pass

       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      (
           "co_argcount",
           "co_kwonlyargcount",
           "co_nlocals",
           "co_stacksize",
           "co_flags",
           "co_code",
           "co_consts",
           "co_names",
           "co_varnames",
           "co_filename",
           "co_name",
           "co_firstlineno",
           "co_lnotab",
           "co_freevars",
           "co_cellvars",
       )

       def _create_code_pyaa(*__UpperCamelCase			):
           if hasattr(args[-3]  ,    "encode"			):
               UpperCamelCase__	     =      list(__UpperCamelCase			)
               UpperCamelCase__	     =      args[-3].encode()  # co_lnotab
               UpperCamelCase__	     =      args[-10].encode()  # co_code
           if hasattr(__UpperCamelCase  ,    "co_posonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(*__UpperCamelCase			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(args[0]  ,    0  ,    *args[1:]			)
               return CodeType(args[0]  ,    0  ,    0  ,    *args[1:]			)
           elif hasattr(__UpperCamelCase  ,    "co_kwonlyargcount"			):
               if len(__UpperCamelCase			) == 16:
                   return CodeType(args[0]  ,    *args[2:]			)
               elif len(__UpperCamelCase			) == 15:
                   return CodeType(*__UpperCamelCase			)
               return CodeType(args[0]  ,    0  ,    *args[1:]			)
           if len(__UpperCamelCase			) == 16:
               return CodeType(args[0]  ,    *args[3:]			)
           elif len(__UpperCamelCase			) == 15:
               return CodeType(args[0]  ,    *args[2:]			)
           return CodeType(*__UpperCamelCase			)

       def create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			):
           def func():
               return returned_obj

           UpperCamelCase__	     =      func.__code__
           # Use _create_code from dill in order to make it work for different python versions
           if datasets.config.PY_VERSION < version.parse("3.8"			):
               UpperCamelCase__	     =      _create_code_pyaa(
                   *[getattr(__UpperCamelCase  ,    __UpperCamelCase			) if k != "co_filename" else co_filename for k in code_args_pyaa]			)
           else:
               UpperCamelCase__	     =      code.replace(co_filename=__UpperCamelCase			)
           return FunctionType(__UpperCamelCase  ,    func.__globals__  ,    func.__name__  ,    func.__defaults__  ,    func.__closure__			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-2-e0383a102aae>", [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      "<ipython-input-5-713f6613acf3>", [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)

       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "321456789.py"			), [1]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       UpperCamelCase__     ,	UpperCamelCase__	     =      os.path.join(gettempdir()  ,    "ipykernel_12345"  ,    "654123987.py"			), [0]
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase  ,    __UpperCamelCase			)			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__     ,	UpperCamelCase__	     =      [0], [1]

       def func():
           return foo, bar

       UpperCamelCase__	     =      "__main__"

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"foo": foo, "bar": bar}

       def globalvars_mocka_side_effect(__UpperCamelCase  ,    *__UpperCamelCase  ,    **__UpperCamelCase			):
           return {"bar": bar, "foo": foo}

       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       with patch("dill.detect.globalvars"  ,    side_effect=__UpperCamelCase			) as globalvars_mocka:
           UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
           self.assertGreater(globalvars_mocka.call_count  ,    0			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("hello"			)
       UpperCamelCase__	     =      Hasher.hash("there"			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("hello"			)			)
       UpperCamelCase__	     =      Hasher.hash(Foo("there"			)			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["hello", Foo("hello"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       UpperCamelCase__	     =      Hasher()
       for x in ["there", Foo("there"			)]:
           hasher.update(__UpperCamelCase			)
       UpperCamelCase__	     =      hasher.hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   def 						_UpperCamelCase							(	self			):
       with self.assertRaises(pickle.PicklingError			):
           Hasher.hash(UnpicklableCallable(Foo("hello"			)			)			)


   def 						_UpperCamelCase							(	self			):
       UpperCamelCase__	     =      "abc"
       UpperCamelCase__	     =      [string, string]  # two strings have the same ids
       UpperCamelCase__	     =      [string, string]
       UpperCamelCase__	     =      json.loads(f'["{string}", "{string}"]'			)  # two strings have different ids
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIs(obja[0]  ,    __UpperCamelCase			)
       self.assertIs(obja[0]  ,    obja[1]			)
       self.assertIsNot(obja[0]  ,    __UpperCamelCase			)
       self.assertIsNot(obja[0]  ,    obja[1]			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       UpperCamelCase__	     =      Hasher.hash(__UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_tiktoken
   def 						_UpperCamelCase							(	self			):
       import tiktoken

       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("r50k_base"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      tiktoken.get_encoding("gpt2"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)


   @require_torch
   def 						_UpperCamelCase							(	self			):
       import torch

       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([2.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      torch.tensor([1.0]			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)







   @require_spacy
   @require_spacy_model("en_core_web_sm"			)
   @require_spacy_model("fr_core_news_sm"			)
   @pytest.mark.integration
   def 						_UpperCamelCase							(	self			):
       import spacy

       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("fr_core_news_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       UpperCamelCase__	     =      spacy.load("en_core_web_sm"			)
       UpperCamelCase__	     =      mda(datasets.utils.py_utils.dumps(__UpperCamelCase			)			).hexdigest()
       self.assertEqual(__UpperCamelCase  ,    __UpperCamelCase			)
       self.assertNotEqual(__UpperCamelCase  ,    __UpperCamelCase			)





@pytest.mark.integration
def     snake_case__			(lowerCAmelCase_						):
    UpperCamelCase__	     =      tmp_path / "dir1"
    UpperCamelCase__	     =      tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    UpperCamelCase__	     =      "script.py"
    UpperCamelCase__	     =      dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
    UpperCamelCase__	     =      dira / script_filename
    UpperCamelCase__	     =      dira / script_filename
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    with script_patha.open("w"						) as f:
        f.write(lowerCAmelCase_						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    UpperCamelCase__	     =      subprocess.check_output(["python", str(lowerCAmelCase_						)]						)
    assert fingerprinta == fingerprinta





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}
    UpperCamelCase__	     =      DatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						)
    UpperCamelCase__	     =      dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2						) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint





def     snake_case__			():
    UpperCamelCase__	     =      {"a": [0, 1, 2]}

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="1.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    class    lowerCamelCase_				(						datasets.Dataset	):






       '''simple docstring'''

       @fingerprint_transform(inplace=__UpperCamelCase  ,    version="2.0.0"			)
       def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
           return DummyDatasetChild(self.data  ,    fingerprint=__UpperCamelCase			)

    UpperCamelCase__	     =      DummyDatasetChild(InMemoryTable.from_pydict(lowerCAmelCase_						)						).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3





def     snake_case__			():
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    UpperCamelCase__	     =      Hasher()
    hasher.update(lambda lowerCAmelCase_						: x						)




