import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class  __lowerCAmelCase	:



 '''simple docstring'''

 def __init__(      self							:		Optional[int]							,      lowerCamelCase							:		Optional[Any]	)			->				Union[str, Any]:
        _UpperCamelCase  :							Union[str, Any]	    =	foo

 def __call__(      self							:		Tuple	)			->				List[str]:
        return self.foo







class  __lowerCAmelCase	(							datasets.Dataset    ):



 '''simple docstring'''

 @fingerprint_transform(inplace=lowerCamelCase	)
 def        __snake_case       (      self							:		Optional[Any]							,      lowerCamelCase							:		str							,      *lowerCamelCase							:		Tuple							,      **lowerCamelCase							:		Optional[int]	)			->				List[Any]:
        return DatasetChild(self.data							,      fingerprint=lowerCamelCase	)

 @fingerprint_transform(inplace=lowerCamelCase	)
 def        __snake_case       (      self							:		Dict							,      lowerCamelCase							:		Any							,      *lowerCamelCase							:		Tuple							,      **lowerCamelCase							:		List[Any]	)			->				Optional[int]:
        return DatasetChild(self.data							,      fingerprint=lowerCamelCase	)







class  __lowerCAmelCase	:



 '''simple docstring'''

 def __init__(      self							:		Tuple							,      lowerCamelCase							:		List[Any]	)			->				Union[str, Any]:
        _UpperCamelCase  :							Tuple	    =	callable

 def __call__(      self							:		int							,      *lowerCamelCase							:		Tuple							,      **lowerCamelCase							:		List[Any]	)			->				Union[str, Any]:
        if self.callable is not None:
               return self.callable(*lowerCamelCase							,      **lowerCamelCase	)

 def __getstate__(      self							:		Optional[Any]	)			->				Tuple:
        raise pickle.PicklingError()







class  __lowerCAmelCase	(							a_    ):



 '''simple docstring'''

 @require_transformers
 @pytest.mark.integration
 def        __snake_case       (      self							:		List[str]	)			->				Dict:
        from transformers import AutoTokenizer

        def encode(lowerCamelCase							:		Optional[Any]	):
               return tokenizer(lowerCamelCase	)

        # TODO: add hash consistency tests across sessions
        _UpperCamelCase  :							int	    =	AutoTokenizer.from_pretrained('bert-base-uncased'	)
        _UpperCamelCase  :							Tuple	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Optional[Any]	    =	mda(datasets.utils.py_utils.dumps(lambda lowerCamelCase	: tokenizer(lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	AutoTokenizer.from_pretrained('bert-base-cased'	)
        _UpperCamelCase  :							Optional[Any]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Optional[Any]	    =	mda(datasets.utils.py_utils.dumps(lambda lowerCamelCase	: tokenizer(lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							int	    =	AutoTokenizer.from_pretrained('bert-base-uncased'	)
        _UpperCamelCase  :							Dict	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Union[str, Any]	    =	mda(datasets.utils.py_utils.dumps(lambda lowerCamelCase	: tokenizer(lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase  :							str	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 @require_transformers
 @pytest.mark.integration
 def        __snake_case       (      self							:		Optional[Any]	)			->				str:
        from transformers import AutoTokenizer

        _UpperCamelCase  :							Union[str, Any]	    =	AutoTokenizer.from_pretrained('gpt2'	)
        _UpperCamelCase  :							Tuple	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        tokenizer('Hello world !'	)  # call once to change the tokenizer's cache
        _UpperCamelCase  :							Union[str, Any]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)

 @require_regex
 def        __snake_case       (      self							:		List[str]	)			->				Tuple:
        import regex

        _UpperCamelCase  :							Optional[int]	    =	regex.Regex('foo'	)
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	regex.Regex('bar'	)
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Optional[int]	    =	regex.Regex('foo'	)
        _UpperCamelCase  :							int	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)







class  __lowerCAmelCase	(							a_    ):



 '''simple docstring'''

 def        __snake_case       (      self							:		Tuple	)			->				Tuple:
        def func():
               return foo

        _UpperCamelCase  :							List[str]	    =	[0]
        _UpperCamelCase  :							Dict	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Dict	    =	[1]
        _UpperCamelCase  :							List[str]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	[0]
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		int	)			->				List[str]:
        def func():
               pass

        _UpperCamelCase  :							List[str]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()

        def func():
               pass

        _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		List[Any]	)			->				List[str]:
        _UpperCamelCase  :							List[Any]	    =	mda(datasets.utils.py_utils.dumps(Foo([0]	)	)	).hexdigest()
        _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(Foo([1]	)	)	).hexdigest()
        _UpperCamelCase  :							str	    =	mda(datasets.utils.py_utils.dumps(Foo([0]	)	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		str	)			->				Dict:
        _UpperCamelCase  :							str	    =	mda(datasets.utils.py_utils.dumps(Foo([0]	).__call__	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	mda(datasets.utils.py_utils.dumps(Foo([1]	).__call__	)	).hexdigest()
        _UpperCamelCase  :							Optional[Any]	    =	mda(datasets.utils.py_utils.dumps(Foo([0]	).__call__	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		List[Any]	)			->				Union[str, Any]:
        _UpperCamelCase  :							int	    =	(
            'co_argcount',
            'co_kwonlyargcount',
            'co_nlocals',
            'co_stacksize',
            'co_flags',
            'co_code',
            'co_consts',
            'co_names',
            'co_varnames',
            'co_filename',
            'co_name',
            'co_firstlineno',
            'co_lnotab',
            'co_freevars',
            'co_cellvars',
        )

        def _create_code_pyaa(*lowerCamelCase							:		List[str]	):
               if hasattr(args[-3]							,      'encode'	):
                      _UpperCamelCase  :							Optional[int]	    =	list(lowerCamelCase	)
                      _UpperCamelCase  :							Optional[int]	    =	args[-3].encode()  # co_lnotab
                      _UpperCamelCase  :							int	    =	args[-1_0].encode()  # co_code
               if hasattr(lowerCamelCase							,      'co_posonlyargcount'	):
                      if len(lowerCamelCase	) == 1_6:
                             return CodeType(*lowerCamelCase	)
                      elif len(lowerCamelCase	) == 1_5:
                             return CodeType(args[0]							,      0							,      *args[1:]	)
                      return CodeType(args[0]							,      0							,      0							,      *args[1:]	)
               elif hasattr(lowerCamelCase							,      'co_kwonlyargcount'	):
                      if len(lowerCamelCase	) == 1_6:
                             return CodeType(args[0]							,      *args[2:]	)
                      elif len(lowerCamelCase	) == 1_5:
                             return CodeType(*lowerCamelCase	)
                      return CodeType(args[0]							,      0							,      *args[1:]	)
               if len(lowerCamelCase	) == 1_6:
                      return CodeType(args[0]							,      *args[3:]	)
               elif len(lowerCamelCase	) == 1_5:
                      return CodeType(args[0]							,      *args[2:]	)
               return CodeType(*lowerCamelCase	)

        def create_ipython_func(lowerCamelCase							:		int							,      lowerCamelCase							:		List[Any]	):
               def func():
                      return returned_obj

               _UpperCamelCase  :							Optional[int]	    =	func.__code__
               # Use _create_code from dill in order to make it work for different python versions
               if datasets.config.PY_VERSION < version.parse('3.8'	):
                      _UpperCamelCase  :							Any	    =	_create_code_pyaa(
                          *[getattr(lowerCamelCase							,      lowerCamelCase	) if k != 'co_filename' else co_filename for k in code_args_pyaa]	)
               else:
                      _UpperCamelCase  :							Optional[int]	    =	code.replace(co_filename=lowerCamelCase	)
               return FunctionType(lowerCamelCase							,      func.__globals__							,      func.__name__							,      func.__defaults__							,      func.__closure__	)

        _UpperCamelCase    ,   _UpperCamelCase  :							List[Any]	    =	'<ipython-input-2-e0383a102aae>', [0]
        _UpperCamelCase  :							Dict	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase    ,   _UpperCamelCase  :							Any	    =	'<ipython-input-2-e0383a102aae>', [1]
        _UpperCamelCase  :							Tuple	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase    ,   _UpperCamelCase  :							str	    =	'<ipython-input-5-713f6613acf3>', [0]
        _UpperCamelCase  :							str	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

        _UpperCamelCase    ,   _UpperCamelCase  :							Union[str, Any]	    =	os.path.join(gettempdir()							,      'ipykernel_12345'							,      '321456789.py'	), [0]
        _UpperCamelCase  :							List[Any]	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase    ,   _UpperCamelCase  :							str	    =	os.path.join(gettempdir()							,      'ipykernel_12345'							,      '321456789.py'	), [1]
        _UpperCamelCase  :							List[Any]	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        _UpperCamelCase    ,   _UpperCamelCase  :							Optional[Any]	    =	os.path.join(gettempdir()							,      'ipykernel_12345'							,      '654123987.py'	), [0]
        _UpperCamelCase  :							int	    =	mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCamelCase							,      lowerCamelCase	)	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		str	)			->				List[Any]:
        _UpperCamelCase    ,   _UpperCamelCase  :							str	    =	[0], [1]

        def func():
               return foo, bar

        _UpperCamelCase  :							int	    =	'__main__'

        def globalvars_mocka_side_effect(lowerCamelCase							:		List[str]							,      *lowerCamelCase							:		List[str]							,      **lowerCamelCase							:		Any	):
               return {"foo": foo, "bar": bar}

        def globalvars_mocka_side_effect(lowerCamelCase							:		int							,      *lowerCamelCase							:		Tuple							,      **lowerCamelCase							:		Any	):
               return {"bar": bar, "foo": foo}

        with patch('dill.detect.globalvars'							,      side_effect=lowerCamelCase	) as globalvars_mocka:
               _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
               self.assertGreater(globalvars_mocka.call_count							,      0	)
        with patch('dill.detect.globalvars'							,      side_effect=lowerCamelCase	) as globalvars_mocka:
               _UpperCamelCase  :							Optional[Any]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
               self.assertGreater(globalvars_mocka.call_count							,      0	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)







class  __lowerCAmelCase	(							a_    ):



 '''simple docstring'''

 def        __snake_case       (      self							:		List[Any]	)			->				str:
        _UpperCamelCase  :							List[Any]	    =	Hasher.hash('hello'	)
        _UpperCamelCase  :							Optional[Any]	    =	Hasher.hash('hello'	)
        _UpperCamelCase  :							Tuple	    =	Hasher.hash('there'	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		Dict	)			->				Dict:
        _UpperCamelCase  :							Tuple	    =	Hasher.hash(Foo('hello'	)	)
        _UpperCamelCase  :							str	    =	Hasher.hash(Foo('hello'	)	)
        _UpperCamelCase  :							int	    =	Hasher.hash(Foo('there'	)	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		List[Any]	)			->				List[Any]:
        _UpperCamelCase  :							Optional[Any]	    =	Hasher()
        for x in ["hello", Foo('hello'	)]:
               hasher.update(lowerCamelCase	)
        _UpperCamelCase  :							Dict	    =	hasher.hexdigest()
        _UpperCamelCase  :							Union[str, Any]	    =	Hasher()
        for x in ["hello", Foo('hello'	)]:
               hasher.update(lowerCamelCase	)
        _UpperCamelCase  :							int	    =	hasher.hexdigest()
        _UpperCamelCase  :							Tuple	    =	Hasher()
        for x in ["there", Foo('there'	)]:
               hasher.update(lowerCamelCase	)
        _UpperCamelCase  :							Union[str, Any]	    =	hasher.hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 def        __snake_case       (      self							:		str	)			->				Tuple:
        with self.assertRaises(pickle.PicklingError	):
               Hasher.hash(UnpicklableCallable(Foo('hello'	)	)	)

 def        __snake_case       (      self							:		Any	)			->				Dict:
        _UpperCamelCase  :							Optional[Any]	    =	'abc'
        _UpperCamelCase  :							List[Any]	    =	[string, string]  # two strings have the same ids
        _UpperCamelCase  :							List[str]	    =	[string, string]
        _UpperCamelCase  :							Tuple	    =	json.loads(f"[\"{string}\", \"{string}\"]"	)  # two strings have different ids
        self.assertIs(obja[0]							,      lowerCamelCase	)
        self.assertIs(obja[0]							,      obja[1]	)
        self.assertIs(obja[0]							,      lowerCamelCase	)
        self.assertIs(obja[0]							,      obja[1]	)
        self.assertIsNot(obja[0]							,      lowerCamelCase	)
        self.assertIsNot(obja[0]							,      obja[1]	)
        _UpperCamelCase  :							Optional[int]	    =	Hasher.hash(lowerCamelCase	)
        _UpperCamelCase  :							str	    =	Hasher.hash(lowerCamelCase	)
        _UpperCamelCase  :							int	    =	Hasher.hash(lowerCamelCase	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)

 @require_tiktoken
 def        __snake_case       (      self							:		Any	)			->				Union[str, Any]:
        import tiktoken

        _UpperCamelCase  :							Optional[Any]	    =	tiktoken.get_encoding('gpt2'	)
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Optional[Any]	    =	tiktoken.get_encoding('r50k_base'	)
        _UpperCamelCase  :							Dict	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							str	    =	tiktoken.get_encoding('gpt2'	)
        _UpperCamelCase  :							List[Any]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 @require_torch
 def        __snake_case       (      self							:		Optional[Any]	)			->				Tuple:
        import torch

        _UpperCamelCase  :							str	    =	torch.tensor([1.0]	)
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Optional[Any]	    =	torch.tensor([2.0]	)
        _UpperCamelCase  :							Union[str, Any]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							Union[str, Any]	    =	torch.tensor([1.0]	)
        _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)

 @require_spacy
 @require_spacy_model('en_core_web_sm'	)
 @require_spacy_model('fr_core_news_sm'	)
 @pytest.mark.integration
 def        __snake_case       (      self							:		Optional[int]	)			->				Tuple:
        import spacy

        _UpperCamelCase  :							List[Any]	    =	spacy.load('en_core_web_sm'	)
        _UpperCamelCase  :							Optional[int]	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	spacy.load('fr_core_news_sm'	)
        _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        _UpperCamelCase  :							List[Any]	    =	spacy.load('en_core_web_sm'	)
        _UpperCamelCase  :							Any	    =	mda(datasets.utils.py_utils.dumps(lowerCamelCase	)	).hexdigest()
        self.assertEqual(lowerCamelCase							,      lowerCamelCase	)
        self.assertNotEqual(lowerCamelCase							,      lowerCamelCase	)







@pytest.mark.integration
def SCREAMING_SNAKE_CASE					(  lowercase_:						Path					):
       _UpperCamelCase  :							str	    =	tmp_path / 'dir1'
       _UpperCamelCase  :							Optional[int]	    =	tmp_path / 'dir2'
       dira.mkdir()
       dira.mkdir()
       _UpperCamelCase  :							Union[str, Any]	    =	'script.py'
       _UpperCamelCase  :							Tuple	    =	dedent(
           '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '					)
       _UpperCamelCase  :							Optional[Any]	    =	dira / script_filename
       _UpperCamelCase  :							Optional[int]	    =	dira / script_filename
       with script_patha.open('w'					) as f:
              f.write(lowercase_					)
       with script_patha.open('w'					) as f:
              f.write(lowercase_					)
       _UpperCamelCase  :							List[Any]	    =	subprocess.check_output(['python', str(lowercase_					)]					)
       _UpperCamelCase  :							str	    =	subprocess.check_output(['python', str(lowercase_					)]					)
       assert fingerprinta == fingerprinta


def SCREAMING_SNAKE_CASE					(  ):
       _UpperCamelCase  :							Union[str, Any]	    =	{'a': [0, 1, 2]}
       _UpperCamelCase  :							List[str]	    =	DatasetChild(InMemoryTable.from_pydict(lowercase_					)					)
       _UpperCamelCase  :							int	    =	dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2					) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca					).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca					).get()._fingerprint


def SCREAMING_SNAKE_CASE					(  ):
       _UpperCamelCase  :							int	    =	{'a': [0, 1, 2]}




       class  __lowerCAmelCase	(							datasets.Dataset    ):



        '''simple docstring'''

        @fingerprint_transform(inplace=lowerCamelCase	)
        def        __snake_case       (      self							:		Any							,      lowerCamelCase							:		Dict	)			->				int:
               return DummyDatasetChild(self.data							,      fingerprint=lowerCamelCase	)







       _UpperCamelCase  :							str	    =	DummyDatasetChild(InMemoryTable.from_pydict(lowercase_					)					).func()




       class  __lowerCAmelCase	(							datasets.Dataset    ):



        '''simple docstring'''

        @fingerprint_transform(inplace=lowerCamelCase							,      version='1.0.0'	)
        def        __snake_case       (      self							:		int							,      lowerCamelCase							:		Optional[Any]	)			->				Tuple:
               return DummyDatasetChild(self.data							,      fingerprint=lowerCamelCase	)







       _UpperCamelCase  :							Optional[int]	    =	DummyDatasetChild(InMemoryTable.from_pydict(lowercase_					)					).func()




       class  __lowerCAmelCase	(							datasets.Dataset    ):



        '''simple docstring'''

        @fingerprint_transform(inplace=lowerCamelCase							,      version='2.0.0'	)
        def        __snake_case       (      self							:		Optional[int]							,      lowerCamelCase							:		Optional[Any]	)			->				Tuple:
               return DummyDatasetChild(self.data							,      fingerprint=lowerCamelCase	)







       _UpperCamelCase  :							Tuple	    =	DummyDatasetChild(InMemoryTable.from_pydict(lowercase_					)					).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}					) == 3





def SCREAMING_SNAKE_CASE					(  ):
       # AttributeError: module 'dill._dill' has no attribute 'stack'
       _UpperCamelCase  :							Union[str, Any]	    =	Hasher()
       hasher.update(lambda lowercase_					: x					)









'''simple docstring'''





import functools
from contextlib import contextmanager

import bert_score
from packaging import version

import datasets





@contextmanager
def UpperCAmelCase    (       ):
    def filter_log(UpperCAmelCase_						):
        return False if "This IS expected if you are initializing" in record.msg else True

    _a				:							Optional[Any]							       =      datasets.utils.logging.get_logger("""transformers.modeling_utils"""						)
    logger.addFilter(UpperCAmelCase_						)
    try:
        yield
    finally:
        logger.removeFilter(UpperCAmelCase_						)


a		:	Tuple       =      '''\
@inproceedings{bert-score,
  title={BERTScore: Evaluating Text Generation with BERT},
  author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi},
  booktitle={International Conference on Learning Representations},
  year={2020},
  url={https://openreview.net/forum?id=SkeHuCVFDr}
}
'''

a		:	List[str]       =      '''\
BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference
sentences by cosine similarity.
It has been shown to correlate with human judgment on sentence-level and system-level evaluation.
Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language
generation tasks.

See the project\'s README at https://github.com/Tiiiger/bert_score#readme for more information.
'''

a		:	Dict       =      '''
BERTScore Metrics with the hashcode from a source against one or more references.

Args:
    predictions (list of str): Prediction/candidate sentences.
    references (list of str or list of list of str): Reference sentences.
    lang (str): Language of the sentences; required (e.g. \'en\').
    model_type (str): Bert specification, default using the suggested
        model for the target language; has to specify at least one of
        `model_type` or `lang`.
    num_layers (int): The layer of representation to use,
        default using the number of layers tuned on WMT16 correlation data.
    verbose (bool): Turn on intermediate status update.
    idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict.
    device (str): On which the contextual embedding model will be allocated on.
        If this argument is None, the model lives on cuda:0 if cuda is available.
    nthreads (int): Number of threads.
    batch_size (int): Bert score processing batch size,
        at least one of `model_type` or `lang`. `lang` needs to be
        specified when `rescale_with_baseline` is True.
    rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline.
    baseline_path (str): Customized baseline file.
    use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10.

Returns:
    precision: Precision.
    recall: Recall.
    f1: F1 score.
    hashcode: Hashcode of the library.

Examples:

    >>> predictions = ["hello there", "general kenobi"]
    >>> references = ["hello there", "general kenobi"]
    >>> bertscore = datasets.load_metric("bertscore")
    >>> results = bertscore.compute(predictions=predictions, references=references, lang="en")
    >>> print([round(v, 2) for v in results["f1"]])
    [1.0, 1.0]
'''





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,		_KWARGS_DESCRIPTION       )
class      UpperCamelCase       (	datasets.Metric       ):



  def     a_							(			self ):







      '''simple docstring'''


      return datasets.MetricInfo(
          description=_DESCRIPTION		,     citation=_CITATION		,     homepage="""https://github.com/Tiiiger/bert_score"""		,     inputs_description=_KWARGS_DESCRIPTION		,     features=datasets.Features(
              {
                  """predictions""": datasets.Value("""string"""		,     id="""sequence""" ),
                  """references""": datasets.Sequence(datasets.Value("""string"""		,     id="""sequence""" )		,     id="""references""" ),
              } )		,     codebase_urls=["""https://github.com/Tiiiger/bert_score"""]		,     reference_urls=[
              """https://github.com/Tiiiger/bert_score""",
              """https://arxiv.org/abs/1904.09675""",
          ]		,     )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     __lowerCAmelCase=False		,     __lowerCAmelCase=False		,     __lowerCAmelCase=None		,     __lowerCAmelCase=64		,     __lowerCAmelCase=4		,     __lowerCAmelCase=False		,     __lowerCAmelCase=False		,     __lowerCAmelCase=None		,     __lowerCAmelCase=False		,     ):







      '''simple docstring'''


      _a				:							Any							       =      bert_score.utils.get_hash
      _a				:							int							       =      bert_score.BERTScorer

      if version.parse(bert_score.__version__ ) >= version.parse("""0.3.10""" ):
          _a				:							Union[str, Any]							       =      functools.partial(__lowerCAmelCase		,     use_fast_tokenizer=__lowerCAmelCase )
          _a				:							Dict							       =      functools.partial(__lowerCAmelCase		,     use_fast_tokenizer=__lowerCAmelCase )
      elif use_fast_tokenizer:
          raise ImportWarning(
              """To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of `bert-score` doesn't match this condition.\n"""
              """You can install it with `pip install \"bert-score>=0.3.10\"`.""" )

      if model_type is None:
          assert lang is not None, "either lang or model_type should be specified"
          _a				:							List[str]							       =      bert_score.utils.langamodel[lang.lower()]

      if num_layers is None:
          _a				:							Dict							       =      bert_score.utils.modelalayers[model_type]

      _a				:							Union[str, Any]							       =      get_hash(
          model=__lowerCAmelCase		,     num_layers=__lowerCAmelCase		,     idf=__lowerCAmelCase		,     rescale_with_baseline=__lowerCAmelCase		,     use_custom_baseline=baseline_path is not None		,     )

      with filter_logging_context():
          if not hasattr(self		,     """cached_bertscorer""" ) or self.cached_bertscorer.hash != hashcode:
              _a				:							Optional[Any]							       =      scorer(
                  model_type=__lowerCAmelCase		,     num_layers=__lowerCAmelCase		,     batch_size=__lowerCAmelCase		,     nthreads=__lowerCAmelCase		,     all_layers=__lowerCAmelCase		,     idf=__lowerCAmelCase		,     device=__lowerCAmelCase		,     lang=__lowerCAmelCase		,     rescale_with_baseline=__lowerCAmelCase		,     baseline_path=__lowerCAmelCase		,     )

      ((_a)       , (_a)       , (_a))				:							Optional[Any]							       =      self.cached_bertscorer.score(
          cands=__lowerCAmelCase		,     refs=__lowerCAmelCase		,     verbose=__lowerCAmelCase		,     batch_size=__lowerCAmelCase		,     )
      _a				:							Tuple							       =      {
          """precision""": P.tolist(),
          """recall""": R.tolist(),
          """f1""": F.tolist(),
          """hashcode""": hashcode,
      }
      return output_dict



  def     a_							(			self		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     **__lowerCAmelCase ):







      '''simple docstring'''


      if references is not None:
          _a				:							Dict							       =      [[ref] if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else ref for ref in references]
      super().add_batch(predictions=__lowerCAmelCase		,     references=__lowerCAmelCase		,     **__lowerCAmelCase )



  def     a_							(			self		,     __lowerCAmelCase=None		,     __lowerCAmelCase=None		,     **__lowerCAmelCase ):







      '''simple docstring'''


      if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ):
          _a				:							Optional[Any]							       =      [reference]
      super().add(prediction=__lowerCAmelCase		,     reference=__lowerCAmelCase		,     **__lowerCAmelCase )


