

'''simple docstring'''

import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)




class 			A_					:






     '''simple docstring'''



     def __init__(						self					:	List[Any]	,     lowercase_					:	List[Any]			)		->      Any:
      UpperCAmelCase			:      Union[str, Any]		   =					foo




     def __call__(						self					:	Optional[int]			)		->      Optional[int]:
      return self.foo




class 			A_					(							datasets.Dataset				):






     '''simple docstring'''



     @fingerprint_transform(inplace=lowercase_			)
     def   UpperCAmelCase_			(						self					:	str	,     lowercase_					:	Tuple	,     *lowercase_					:	Optional[Any]	,     **lowercase_					:	Tuple			)		->      str:
      return DatasetChild(self.data	,     fingerprint=lowercase_			)




     @fingerprint_transform(inplace=lowercase_			)
     def   UpperCAmelCase_			(						self					:	Any	,     lowercase_					:	Optional[int]	,     *lowercase_					:	List[Any]	,     **lowercase_					:	List[Any]			)		->      Tuple:
      return DatasetChild(self.data	,     fingerprint=lowercase_			)




class 			A_					:






     '''simple docstring'''



     def __init__(						self					:	int	,     lowercase_					:	Optional[int]			)		->      Optional[Any]:
      UpperCAmelCase			:      Dict		   =					callable


     def __call__(						self					:	int	,     *lowercase_					:	Dict	,     **lowercase_					:	Optional[Any]			)		->      Optional[int]:
      if self.callable is not None:
       return self.callable(*lowercase_	,     **lowercase_			)




     def __getstate__(						self					:	Dict			)		->      Union[str, Any]:
      raise pickle.PicklingError()




class 			A_					(							_snake_case				):






     '''simple docstring'''



     @require_transformers
     @pytest.mark.integration
     def   UpperCAmelCase_			(						self					:	Any			)		->      str:
      from transformers import AutoTokenizer

      def encode(lowercase_					:	Tuple			):
       return tokenizer(lowercase_			)

      # TODO: add hash consistency tests across sessions
      UpperCAmelCase			:      Union[str, Any]		   =					AutoTokenizer.from_pretrained('bert-base-uncased'			)
      UpperCAmelCase			:      Optional[Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      List[Any]		   =					mda(datasets.utils.py_utils.dumps(lambda lowercase_			: tokenizer(lowercase_			)			)			).hexdigest()
      UpperCAmelCase			:      List[Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Optional[int]		   =					AutoTokenizer.from_pretrained('bert-base-cased'			)
      UpperCAmelCase			:      Tuple		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Optional[Any]		   =					mda(datasets.utils.py_utils.dumps(lambda lowercase_			: tokenizer(lowercase_			)			)			).hexdigest()
      UpperCAmelCase			:      Optional[int]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Tuple		   =					AutoTokenizer.from_pretrained('bert-base-uncased'			)
      UpperCAmelCase			:      Tuple		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Union[str, Any]		   =					mda(datasets.utils.py_utils.dumps(lambda lowercase_			: tokenizer(lowercase_			)			)			).hexdigest()
      UpperCAmelCase			:      str		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     @require_transformers
     @pytest.mark.integration
     def   UpperCAmelCase_			(						self					:	List[str]			)		->      int:
      from transformers import AutoTokenizer

      UpperCAmelCase			:      Optional[Any]		   =					AutoTokenizer.from_pretrained('gpt2'			)
      UpperCAmelCase			:      Union[str, Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      tokenizer('Hello world !'			)  # call once to change the tokenizer's cache
      UpperCAmelCase			:      Dict		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)




     @require_regex
     def   UpperCAmelCase_			(						self					:	Any			)		->      List[str]:
      import regex

      UpperCAmelCase			:      Any		   =					regex.Regex('foo'			)
      UpperCAmelCase			:      Optional[int]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Optional[Any]		   =					regex.Regex('bar'			)
      UpperCAmelCase			:      int		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Union[str, Any]		   =					regex.Regex('foo'			)
      UpperCAmelCase			:      int		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)




class 			A_					(							_snake_case				):






     '''simple docstring'''



     def   UpperCAmelCase_			(						self					:	List[Any]			)		->      List[str]:
      def func():
       return foo

      UpperCAmelCase			:      Dict		   =					[0]
      UpperCAmelCase			:      Any		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      List[str]		   =					[1]
      UpperCAmelCase			:      Optional[Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Any		   =					[0]
      UpperCAmelCase			:      Any		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	int			)		->      List[Any]:
      def func():
       pass

      UpperCAmelCase			:      Tuple		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()

      def func():
       pass

      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	Optional[int]			)		->      Optional[int]:
      UpperCAmelCase			:      Optional[int]		   =					mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
      UpperCAmelCase			:      Optional[int]		   =					mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
      UpperCAmelCase			:      List[Any]		   =					mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	Any			)		->      Dict:
      UpperCAmelCase			:      Tuple		   =					mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
      UpperCAmelCase			:      Dict		   =					mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
      UpperCAmelCase			:      Union[str, Any]		   =					mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	int			)		->      Tuple:
      UpperCAmelCase			:      Any		   =					(
          'co_argcount',
          'co_kwonlyargcount',
          'co_nlocals',
          'co_stacksize',
          'co_flags',
          'co_code',
          'co_consts',
          'co_names',
          'co_varnames',
          'co_filename',
          'co_name',
          'co_firstlineno',
          'co_lnotab',
          'co_freevars',
          'co_cellvars',
      )

      def _create_code_pyaa(*lowercase_					:	Any			):
       if hasattr(args[-3]	,     'encode'			):
        UpperCAmelCase			:      Optional[Any]		   =					list(lowercase_			)
        UpperCAmelCase			:      Tuple		   =					args[-3].encode()  # co_lnotab
        UpperCAmelCase			:      Tuple		   =					args[-10].encode()  # co_code
       if hasattr(lowercase_	,     'co_posonlyargcount'			):
        if len(lowercase_			) == 16:
         return CodeType(*lowercase_			)
        elif len(lowercase_			) == 15:
         return CodeType(args[0]	,     0	,     *args[1:]			)
        return CodeType(args[0]	,     0	,     0	,     *args[1:]			)
       elif hasattr(lowercase_	,     'co_kwonlyargcount'			):
        if len(lowercase_			) == 16:
         return CodeType(args[0]	,     *args[2:]			)
        elif len(lowercase_			) == 15:
         return CodeType(*lowercase_			)
        return CodeType(args[0]	,     0	,     *args[1:]			)
       if len(lowercase_			) == 16:
        return CodeType(args[0]	,     *args[3:]			)
       elif len(lowercase_			) == 15:
        return CodeType(args[0]	,     *args[2:]			)
       return CodeType(*lowercase_			)

      def create_ipython_func(lowercase_					:	List[Any]	,     lowercase_					:	Optional[Any]			):
       def func():
        return returned_obj

       UpperCAmelCase			:      Dict		   =					func.__code__
       # Use _create_code from dill in order to make it work for different python versions
       if datasets.config.PY_VERSION < version.parse('3.8'			):
        UpperCAmelCase			:      List[str]		   =					_create_code_pyaa(
            *[getattr(lowercase_	,     lowercase_			) if k != 'co_filename' else co_filename for k in code_args_pyaa]			)
       else:
        UpperCAmelCase			:      Any		   =					code.replace(co_filename=lowercase_			)
       return FunctionType(lowercase_	,     func.__globals__	,     func.__name__	,     func.__defaults__	,     func.__closure__			)

      UpperCAmelCase   ,					UpperCAmelCase			:      List[str]		   =					'<ipython-input-2-e0383a102aae>', [0]
      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      UpperCAmelCase   ,					UpperCAmelCase			:      Optional[int]		   =					'<ipython-input-2-e0383a102aae>', [1]
      UpperCAmelCase			:      Optional[Any]		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      UpperCAmelCase   ,					UpperCAmelCase			:      Optional[Any]		   =					'<ipython-input-5-713f6613acf3>', [0]
      UpperCAmelCase			:      List[Any]		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)

      UpperCAmelCase   ,					UpperCAmelCase			:      Union[str, Any]		   =					os.path.join(gettempdir()	,     'ipykernel_12345'	,     '321456789.py'			), [0]
      UpperCAmelCase			:      str		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      UpperCAmelCase   ,					UpperCAmelCase			:      Optional[int]		   =					os.path.join(gettempdir()	,     'ipykernel_12345'	,     '321456789.py'			), [1]
      UpperCAmelCase			:      Any		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      UpperCAmelCase   ,					UpperCAmelCase			:      Optional[Any]		   =					os.path.join(gettempdir()	,     'ipykernel_12345'	,     '654123987.py'			), [0]
      UpperCAmelCase			:      int		   =					mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_	,     lowercase_			)			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)




     def   UpperCAmelCase_			(						self					:	str			)		->      List[Any]:
      UpperCAmelCase   ,					UpperCAmelCase			:      Union[str, Any]		   =					[0], [1]

      def func():
       return foo, bar

      UpperCAmelCase			:      Any		   =					'__main__'

      def globalvars_mocka_side_effect(lowercase_					:	Union[str, Any]	,     *lowercase_					:	List[str]	,     **lowercase_					:	Union[str, Any]			):
       return {"foo": foo, "bar": bar}

      def globalvars_mocka_side_effect(lowercase_					:	List[Any]	,     *lowercase_					:	Optional[int]	,     **lowercase_					:	List[Any]			):
       return {"bar": bar, "foo": foo}

      with patch('dill.detect.globalvars'	,     side_effect=lowercase_			) as globalvars_mocka:
       UpperCAmelCase			:      int		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count	,     0			)
      with patch('dill.detect.globalvars'	,     side_effect=lowercase_			) as globalvars_mocka:
       UpperCAmelCase			:      Any		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count	,     0			)
      self.assertEqual(lowercase_	,     lowercase_			)




class 			A_					(							_snake_case				):






     '''simple docstring'''



     def   UpperCAmelCase_			(						self					:	Optional[int]			)		->      List[Any]:
      UpperCAmelCase			:      Tuple		   =					Hasher.hash('hello'			)
      UpperCAmelCase			:      List[Any]		   =					Hasher.hash('hello'			)
      UpperCAmelCase			:      List[Any]		   =					Hasher.hash('there'			)
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	Dict			)		->      List[Any]:
      UpperCAmelCase			:      Tuple		   =					Hasher.hash(Foo('hello'			)			)
      UpperCAmelCase			:      str		   =					Hasher.hash(Foo('hello'			)			)
      UpperCAmelCase			:      List[str]		   =					Hasher.hash(Foo('there'			)			)
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	List[str]			)		->      List[Any]:
      UpperCAmelCase			:      Any		   =					Hasher()
      for x in ["hello", Foo('hello'			)]:
       hasher.update(lowercase_			)
      UpperCAmelCase			:      Optional[int]		   =					hasher.hexdigest()
      UpperCAmelCase			:      str		   =					Hasher()
      for x in ["hello", Foo('hello'			)]:
       hasher.update(lowercase_			)
      UpperCAmelCase			:      Tuple		   =					hasher.hexdigest()
      UpperCAmelCase			:      Optional[Any]		   =					Hasher()
      for x in ["there", Foo('there'			)]:
       hasher.update(lowercase_			)
      UpperCAmelCase			:      Optional[int]		   =					hasher.hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     def   UpperCAmelCase_			(						self					:	List[Any]			)		->      Any:
      with self.assertRaises(pickle.PicklingError			):
       Hasher.hash(UnpicklableCallable(Foo('hello'			)			)			)


     def   UpperCAmelCase_			(						self					:	Optional[int]			)		->      str:
      UpperCAmelCase			:      Optional[Any]		   =					'abc'
      UpperCAmelCase			:      Tuple		   =					[string, string]  # two strings have the same ids
      UpperCAmelCase			:      Dict		   =					[string, string]
      UpperCAmelCase			:      Dict		   =					json.loads(f"""[\"{string}\", \"{string}\"]"""			)  # two strings have different ids
      self.assertIs(obja[0]	,     lowercase_			)
      self.assertIs(obja[0]	,     obja[1]			)
      self.assertIs(obja[0]	,     lowercase_			)
      self.assertIs(obja[0]	,     obja[1]			)
      self.assertIsNot(obja[0]	,     lowercase_			)
      self.assertIsNot(obja[0]	,     obja[1]			)
      UpperCAmelCase			:      int		   =					Hasher.hash(lowercase_			)
      UpperCAmelCase			:      List[Any]		   =					Hasher.hash(lowercase_			)
      UpperCAmelCase			:      Optional[int]		   =					Hasher.hash(lowercase_			)
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertEqual(lowercase_	,     lowercase_			)


     @require_tiktoken
     def   UpperCAmelCase_			(						self					:	Optional[Any]			)		->      str:
      import tiktoken

      UpperCAmelCase			:      Union[str, Any]		   =					tiktoken.get_encoding('gpt2'			)
      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Optional[int]		   =					tiktoken.get_encoding('r50k_base'			)
      UpperCAmelCase			:      Union[str, Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      str		   =					tiktoken.get_encoding('gpt2'			)
      UpperCAmelCase			:      Optional[int]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)


     @require_torch
     def   UpperCAmelCase_			(						self					:	Union[str, Any]			)		->      int:
      import torch

      UpperCAmelCase			:      Tuple		   =					torch.tensor([1.0]			)
      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      List[Any]		   =					torch.tensor([2.0]			)
      UpperCAmelCase			:      Optional[Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      str		   =					torch.tensor([1.0]			)
      UpperCAmelCase			:      Union[str, Any]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)




     @require_spacy
     @require_spacy_model('en_core_web_sm'			)
     @require_spacy_model('fr_core_news_sm'			)
     @pytest.mark.integration
     def   UpperCAmelCase_			(						self					:	List[str]			)		->      Dict:
      import spacy

      UpperCAmelCase			:      Optional[int]		   =					spacy.load('en_core_web_sm'			)
      UpperCAmelCase			:      Tuple		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Tuple		   =					spacy.load('fr_core_news_sm'			)
      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      UpperCAmelCase			:      Any		   =					spacy.load('en_core_web_sm'			)
      UpperCAmelCase			:      List[str]		   =					mda(datasets.utils.py_utils.dumps(lowercase_			)			).hexdigest()
      self.assertEqual(lowercase_	,     lowercase_			)
      self.assertNotEqual(lowercase_	,     lowercase_			)




@pytest.mark.integration
def  UpperCamelCase(	UpperCAmelCase_      ):
 UpperCAmelCase			:      Optional[Any]		   =					tmp_path / 'dir1'
 UpperCAmelCase			:      Optional[Any]		   =					tmp_path / 'dir2'
 dira.mkdir()
 dira.mkdir()
 UpperCAmelCase			:      Any		   =					'script.py'
 UpperCAmelCase			:      Any		   =					dedent(
     '\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    '      )
 UpperCAmelCase			:      str		   =					dira / script_filename
 UpperCAmelCase			:      Optional[Any]		   =					dira / script_filename
 with script_patha.open('w'      ) as f:
  f.write(UpperCAmelCase_      )
 with script_patha.open('w'      ) as f:
  f.write(UpperCAmelCase_      )
 UpperCAmelCase			:      Any		   =					subprocess.check_output(['python', str(UpperCAmelCase_      )]      )
 UpperCAmelCase			:      str		   =					subprocess.check_output(['python', str(UpperCAmelCase_      )]      )
 assert fingerprinta == fingerprinta




def  UpperCamelCase(	):
 UpperCAmelCase			:      Optional[int]		   =					{'a': [0, 1, 2]}
 UpperCAmelCase			:      Union[str, Any]		   =					DatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      )
 UpperCAmelCase			:      List[Any]		   =					dataset.funca()._fingerprint
 assert expected_fingerprint == dataset.funca()._fingerprint
 assert expected_fingerprint != dataset.funca()._fingerprint

 with Pool(2      ) as p:
  assert expected_fingerprint == p.apply_async(dataset.funca      ).get()._fingerprint
  assert expected_fingerprint != p.apply_async(dataset.funca      ).get()._fingerprint




def  UpperCamelCase(	):
 UpperCAmelCase			:      Optional[int]		   =					{'a': [0, 1, 2]}




 class 			A_					(							datasets.Dataset				):






      '''simple docstring'''



      @fingerprint_transform(inplace=lowercase_			)
      def   UpperCAmelCase_			(						self					:	Union[str, Any]	,     lowercase_					:	Dict			)		->      Optional[int]:
       return DummyDatasetChild(self.data	,     fingerprint=lowercase_			)



 UpperCAmelCase			:      Optional[int]		   =					DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()




 class 			A_					(							datasets.Dataset				):






      '''simple docstring'''



      @fingerprint_transform(inplace=lowercase_	,     version='1.0.0'			)
      def   UpperCAmelCase_			(						self					:	Dict	,     lowercase_					:	Tuple			)		->      Union[str, Any]:
       return DummyDatasetChild(self.data	,     fingerprint=lowercase_			)



 UpperCAmelCase			:      Dict		   =					DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()




 class 			A_					(							datasets.Dataset				):






      '''simple docstring'''



      @fingerprint_transform(inplace=lowercase_	,     version='2.0.0'			)
      def   UpperCAmelCase_			(						self					:	Union[str, Any]	,     lowercase_					:	Optional[Any]			)		->      Tuple:
       return DummyDatasetChild(self.data	,     fingerprint=lowercase_			)



 UpperCAmelCase			:      Union[str, Any]		   =					DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase_      )      ).func()

 assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}      ) == 3




def  UpperCamelCase(	):
 # AttributeError: module 'dill._dill' has no attribute 'stack'
 UpperCAmelCase			:      Tuple		   =					Hasher()
 hasher.update(lambda UpperCAmelCase_      : x      )







'''simple docstring'''

import sacrebleu as scb
from packaging import version

import datasets


lowercase__            =     "\\n@inproceedings{post-2018-call,\n    title = \"A Call for Clarity in Reporting {BLEU} Scores\",\n    author = \"Post, Matt\",\n    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",\n    month = oct,\n    year = \"2018\",\n    address = \"Belgium, Brussels\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W18-6319\",\n    pages = \"186--191\",\n}\n"

lowercase__            =     "\\nSacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.\nInspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.\nIt also knows all the standard test sets and handles downloading, processing, and tokenization for you.\n\nSee the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.\n"

lowercase__            =     "\nProduces BLEU scores along with its sufficient statistics\nfrom a source against one or more references.\n\nArgs:\n    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.\n    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).\n    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:\n        - `'none'`: no smoothing\n        - `'floor'`: increment zero counts\n        - `'add-k'`: increment num/denom by k for n>1\n        - `'exp'`: exponential decay\n    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).\n    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:\n        - `'none'`: No tokenization.\n        - `'zh'`: Chinese tokenization.\n        - `'13a'`: mimics the `mteval-v13a` script from Moses.\n        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses\n        - `'char'`: Language-agnostic character-level tokenization.\n        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).\n    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.\n    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.\n    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.\n\nReturns:\n    'score': BLEU score,\n    'counts': Counts,\n    'totals': Totals,\n    'precisions': Precisions,\n    'bp': Brevity penalty,\n    'sys_len': predictions length,\n    'ref_len': reference length,\n\nExamples:\n\n    Example 1:\n        >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]\n        >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]\n        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")\n        >>> results = sacrebleu.compute(predictions=predictions, references=references)\n        >>> print(list(results.keys()))\n        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']\n        >>> print(round(results[\"score\"], 1))\n        100.0\n\n    Example 2:\n        >>> predictions = [\"hello there general kenobi\",\n        ...                 \"on our way to ankh morpork\"]\n        >>> references = [[\"hello there general kenobi\", \"hello there !\"],\n        ...                 [\"goodbye ankh morpork\", \"ankh morpork\"]]\n        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")\n        >>> results = sacrebleu.compute(predictions=predictions,\n        ...                             references=references)\n        >>> print(list(results.keys()))\n        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']\n        >>> print(round(results[\"score\"], 1))\n        39.8\n"




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,  _KWARGS_DESCRIPTION				)
class 			A_					(							datasets.Metric				):






     '''simple docstring'''



     def   UpperCAmelCase_			(						self					:	Tuple			)		->      Union[str, Any]:
      if version.parse(scb.__version__			) < version.parse('1.4.12'			):
       raise ImportWarning(
           'To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'
           'You can install it with `pip install "sacrebleu>=1.4.12"`.'			)
      return datasets.MetricInfo(
          description=_DESCRIPTION	,     citation=_CITATION	,     homepage='https://github.com/mjpost/sacreBLEU'	,     inputs_description=_KWARGS_DESCRIPTION	,     features=datasets.Features(
              {
                  'predictions': datasets.Value('string'	,     id='sequence'			),
                  'references': datasets.Sequence(datasets.Value('string'	,     id='sequence'			)	,     id='references'			),
              }			)	,     codebase_urls=['https://github.com/mjpost/sacreBLEU']	,     reference_urls=[
              'https://github.com/mjpost/sacreBLEU',
              'https://en.wikipedia.org/wiki/BLEU',
              'https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213',
          ]	,     )




     def   UpperCAmelCase_			(						self					:	Union[str, Any]	,     lowercase_					:	Union[str, Any]	,     lowercase_					:	int	,     lowercase_					:	List[Any]="exp"	,     lowercase_					:	Optional[int]=None	,     lowercase_					:	List[str]=False	,     lowercase_					:	Tuple=False	,     lowercase_					:	int=None	,     lowercase_					:	str=False	,     )		->      Dict:
      UpperCAmelCase			:      Union[str, Any]		   =					len(references[0]			)
      if any(len(lowercase_			) != references_per_prediction for refs in references			):
       raise ValueError('Sacrebleu requires the same number of references for each prediction'			)
      UpperCAmelCase			:      Optional[Any]		   =					[[refs[i] for refs in references] for i in range(lowercase_			)]
      UpperCAmelCase			:      List[Any]		   =					scb.corpus_bleu(
          lowercase_	,     lowercase_	,     smooth_method=lowercase_	,     smooth_value=lowercase_	,     force=lowercase_	,     lowercase=lowercase_	,     use_effective_order=lowercase_	,     **({'tokenize': tokenize} if tokenize else {})	,     )
      UpperCAmelCase			:      Union[str, Any]		   =					{
          'score': output.score,
          'counts': output.counts,
          'totals': output.totals,
          'precisions': output.precisions,
          'bp': output.bp,
          'sys_len': output.sys_len,
          'ref_len': output.ref_len,
      }
      return output_dict




