





import sacrebleu as scb
from packaging import version

import datasets


_SCREAMING_SNAKE_CASE   						=		"""\
@inproceedings{post-2018-call,
    title = \"A Call for Clarity in Reporting {BLEU} Scores\",
    author = \"Post, Matt\",
    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",
    month = oct,
    year = \"2018\",
    address = \"Belgium, Brussels\",
    publisher = \"Association for Computational Linguistics\",
    url = \"https://www.aclweb.org/anthology/W18-6319\",
    pages = \"186--191\",
}
"""

_SCREAMING_SNAKE_CASE   						=		"""\
SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores.
Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text.
It also knows all the standard test sets and handles downloading, processing, and tokenization for you.

See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information.
"""

_SCREAMING_SNAKE_CASE   						=		"""
Produces BLEU scores along with its sufficient statistics
from a source against one or more references.

Args:
    predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens.
    references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length).
    smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are:
        - `'none'`: no smoothing
        - `'floor'`: increment zero counts
        - `'add-k'`: increment num/denom by k for n>1
        - `'exp'`: exponential decay
    smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`).
    tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are:
        - `'none'`: No tokenization.
        - `'zh'`: Chinese tokenization.
        - `'13a'`: mimics the `mteval-v13a` script from Moses.
        - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses
        - `'char'`: Language-agnostic character-level tokenization.
        - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3).
    lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`.
    force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`.
    use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`.

Returns:
    'score': BLEU score,
    'counts': Counts,
    'totals': Totals,
    'precisions': Precisions,
    'bp': Brevity penalty,
    'sys_len': predictions length,
    'ref_len': reference length,

Examples:

    Example 1:
        >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"], [\"foo bar foobar\", \"foo bar foobar\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions, references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        100.0

    Example 2:
        >>> predictions = [\"hello there general kenobi\",
        ...                 \"on our way to ankh morpork\"]
        >>> references = [[\"hello there general kenobi\", \"hello there !\"],
        ...                 [\"goodbye ankh morpork\", \"ankh morpork\"]]
        >>> sacrebleu = datasets.load_metric(\"sacrebleu\")
        >>> results = sacrebleu.compute(predictions=predictions,
        ...                             references=references)
        >>> print(list(results.keys()))
        ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len']
        >>> print(round(results[\"score\"], 1))
        39.8
"""
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,					_KWARGS_DESCRIPTION       )
class SCREAMING_SNAKE_CASE_    (       datasets.Metric       ):






						def UpperCAmelCase_					(   self   :   str						)				->     List[str]:







							"""simple docstring"""


							if version.parse(scb.__version__						) < version.parse('1.4.12'						):
								raise ImportWarning(
								    'To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn\'t match this condition.\n'
								    'You can install it with `pip install "sacrebleu>=1.4.12"`.'						)
							return datasets.MetricInfo(
							    description=_DESCRIPTION		, citation=_CITATION		, homepage='https://github.com/mjpost/sacreBLEU'		, inputs_description=_KWARGS_DESCRIPTION		, features=datasets.Features(
							        {
							            'predictions': datasets.Value('string'		, id='sequence'						),
							            'references': datasets.Sequence(datasets.Value('string'		, id='sequence'						)		, id='references'						),
							        }						)		, codebase_urls=['https://github.com/mjpost/sacreBLEU']		, reference_urls=[
							        'https://github.com/mjpost/sacreBLEU',
							        'https://en.wikipedia.org/wiki/BLEU',
							        'https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213',
							    ]		, )






						def UpperCAmelCase_					(   self   :   List[Any]		, _A   :   List[str]		, _A   :   Optional[Any]		, _A   :   Tuple="exp"		, _A   :   Tuple=None		, _A   :   Optional[Any]=False		, _A   :   Dict=False		, _A   :   List[str]=None		, _A   :   Dict=False		, )				->     Optional[Any]:







							"""simple docstring"""


							snake_case_		:      List[str]           =						len(references[0]						)
							if any(len(_A						) != references_per_prediction for refs in references						):
								raise ValueError('Sacrebleu requires the same number of references for each prediction'						)
							snake_case_		:      Optional[Any]           =						[[refs[i] for refs in references] for i in range(_A						)]
							snake_case_		:      Any           =						scb.corpus_bleu(
							    _A		, _A		, smooth_method=_A		, smooth_value=_A		, force=_A		, lowercase=_A		, use_effective_order=_A		, **({'tokenize': tokenize} if tokenize else {})		, )
							snake_case_		:      int           =						{
							    'score': output.score,
							    'counts': output.counts,
							    'totals': output.totals,
							    'precisions': output.precisions,
							    'bp': output.bp,
							    'sys_len': output.sys_len,
							    'ref_len': output.ref_len,
							}
							return output_dict








import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTNeoConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
				import jax
				import jax.numpy as jnp

				from transformers.modeling_flax_pytorch_utils import (
				    convert_pytorch_state_dict_to_flax,
				    load_flax_weights_in_pytorch_model,
				)
				from transformers.models.gpt_neo.modeling_flax_gpt_neo import FlaxGPTNeoForCausalLM, FlaxGPTNeoModel

if is_torch_available():
				import torch
class 							snake_case_		:







		def __init__(		self      ,  __lowercase      ,  __lowercase=1_4      ,  __lowercase=7      ,  __lowercase=True      ,  __lowercase=True      ,  __lowercase=False      ,  __lowercase=True      ,  __lowercase=9_9      ,  __lowercase=3_2      ,  __lowercase=4      ,  __lowercase=4      ,  __lowercase=[[["global", "local"], 2]]      ,  __lowercase=3_7      ,  __lowercase="gelu"      ,  __lowercase=0.1      ,  __lowercase=0.1      ,  __lowercase=5_1_2      ,  __lowercase=7      ,  __lowercase=0.0_2      ,  )					-> Tuple:
				lowerCamelCase :			Optional[Any]    		=parent
				lowerCamelCase :			Optional[int]    		=batch_size
				lowerCamelCase :			Any    		=seq_length
				lowerCamelCase :			Optional[Any]    		=is_training
				lowerCamelCase :			Any    		=use_input_mask
				lowerCamelCase :			int    		=use_token_type_ids
				lowerCamelCase :			int    		=use_labels
				lowerCamelCase :			str    		=vocab_size
				lowerCamelCase :			Optional[int]    		=hidden_size
				lowerCamelCase :			int    		=num_hidden_layers
				lowerCamelCase :			List[Any]    		=num_attention_heads
				lowerCamelCase :			Any    		=attention_types
				lowerCamelCase :			List[str]    		=intermediate_size
				lowerCamelCase :			Dict    		=hidden_act
				lowerCamelCase :			Union[str, Any]    		=hidden_dropout_prob
				lowerCamelCase :			Optional[int]    		=attention_probs_dropout_prob
				lowerCamelCase :			Any    		=max_position_embeddings
				lowerCamelCase :			int    		=window_size
				lowerCamelCase :			List[Any]    		=initializer_range
				lowerCamelCase :			Any    		=None
				lowerCamelCase :			List[Any]    		=vocab_size - 1
				lowerCamelCase :			str    		=vocab_size - 1
				lowerCamelCase :			int    		=vocab_size - 1







		def       __lowercase (		self )					-> Dict:
				lowerCamelCase :			List[Any]    		=ids_tensor([self.batch_size, self.seq_length]      ,  self.vocab_size )

				lowerCamelCase :			Tuple    		=None
				if self.use_input_mask:
						lowerCamelCase :			Union[str, Any]    		=random_attention_mask([self.batch_size, self.seq_length] )

				lowerCamelCase :			List[Any]    		=GPTNeoConfig(
				    vocab_size=self.vocab_size      ,  hidden_size=self.hidden_size      ,  num_layers=self.num_hidden_layers      ,  num_heads=self.num_attention_heads      ,  max_position_embeddings=self.max_position_embeddings      ,  use_cache=__lowercase      ,  bos_token_id=self.bos_token_id      ,  eos_token_id=self.eos_token_id      ,  pad_token_id=self.pad_token_id      ,  window_size=self.window_size      ,  attention_types=self.attention_types      ,  )

				return (config, input_ids, input_mask)







		def       __lowercase (		self )					-> Any:
				lowerCamelCase :			List[Any]    		=self.prepare_config_and_inputs()
				lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Optional[Any]    		=config_and_inputs
				lowerCamelCase :			List[str]    		={'''input_ids''': input_ids, '''attention_mask''': attention_mask}
				return config, inputs_dict







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )					-> List[Any]:
				lowerCamelCase :			Tuple    		=2_0
				lowerCamelCase :			Union[str, Any]    		=model_class_name(__lowercase )

				lowerCamelCase :			List[Any]    		=model.init_cache(input_ids.shape[0]      ,  __lowercase )
				lowerCamelCase :			str    		=jnp.ones((input_ids.shape[0], max_decoder_length)      ,  dtype='''i4''' )

				lowerCamelCase :			Optional[int]    		=jnp.broadcast_to(
				    jnp.arange(input_ids.shape[-1] - 1 )[None, :]      ,  (input_ids.shape[0], input_ids.shape[-1] - 1) )
				lowerCamelCase :			str    		=model(
				    input_ids[:, :-1]      ,  attention_mask=__lowercase      ,  past_key_values=__lowercase      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			List[str]    		=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]      ,  dtype='''i4''' )
				lowerCamelCase :			List[Any]    		=model(
				    input_ids[:, -1:]      ,  attention_mask=__lowercase      ,  past_key_values=outputs_cache.past_key_values      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			Any    		=model(__lowercase )

				lowerCamelCase :			Dict    		=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
				self.parent.assertTrue(diff < 1e-3      ,  msg=F"Max diff is {diff}" )







		def       __lowercase (		self      ,  __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )					-> Dict:
				lowerCamelCase :			str    		=2_0
				lowerCamelCase :			Optional[int]    		=model_class_name(__lowercase )

				lowerCamelCase :			int    		=jnp.concatenate(
				    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1]) )]      ,  axis=-1      ,  )

				lowerCamelCase :			Union[str, Any]    		=model.init_cache(input_ids.shape[0]      ,  __lowercase )
				lowerCamelCase :			List[Any]    		=jnp.broadcast_to(
				    jnp.arange(input_ids.shape[-1] - 1 )[None, :]      ,  (input_ids.shape[0], input_ids.shape[-1] - 1) )

				lowerCamelCase :			Optional[Any]    		=model(
				    input_ids[:, :-1]      ,  attention_mask=__lowercase      ,  past_key_values=__lowercase      ,  position_ids=__lowercase      ,  )
				lowerCamelCase :			List[Any]    		=jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]      ,  dtype='''i4''' )
				lowerCamelCase :			Union[str, Any]    		=model(
				    input_ids[:, -1:]      ,  past_key_values=outputs_cache.past_key_values      ,  attention_mask=__lowercase      ,  position_ids=__lowercase      ,  )

				lowerCamelCase :			Union[str, Any]    		=model(__lowercase      ,  attention_mask=__lowercase )

				lowerCamelCase :			Dict    		=np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]) ) )
				self.parent.assertTrue(diff < 1e-3      ,  msg=F"Max diff is {diff}" )

@require_flax
class 							snake_case_		(      _A    ,       _A    ,       unittest.TestCase):
		lowerCamelCase     :List[Any]							=  (FlaxGPTNeoModel, FlaxGPTNeoForCausalLM) if is_flax_available() else ()
		lowerCamelCase     :Union[str, Any]							=  (FlaxGPTNeoForCausalLM,) if is_flax_available() else ()







		def       __lowercase (		self )					-> Union[str, Any]:
				lowerCamelCase :			str    		=FlaxGPTNeoModelTester(self )







		def       __lowercase (		self )					-> Dict:
				for model_class_name in self.all_model_classes:
						lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Dict    		=self.model_tester.prepare_config_and_inputs()
						self.model_tester.check_use_cache_forward(__lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )







		def       __lowercase (		self )					-> Tuple:
				for model_class_name in self.all_model_classes:
						lowerCamelCase		, lowerCamelCase		, lowerCamelCase :			Optional[int]    		=self.model_tester.prepare_config_and_inputs()
						self.model_tester.check_use_cache_forward_with_attn_mask(
						    __lowercase      ,  __lowercase      ,  __lowercase      ,  __lowercase )







		@slow
		def       __lowercase (		self )					-> Optional[int]:
				lowerCamelCase :			Tuple    		=GPTaTokenizer.from_pretrained('''gpt2'''      ,  pad_token='''<|endoftext|>'''      ,  padding_side='''left''' )
				lowerCamelCase :			Any    		=tokenizer(['''Hello this is a long string''', '''Hey''']      ,  return_tensors='''np'''      ,  padding=__lowercase      ,  truncation=__lowercase )

				lowerCamelCase :			List[Any]    		=FlaxGPTNeoForCausalLM.from_pretrained('''EleutherAI/gpt-neo-125M''' )
				lowerCamelCase :			Union[str, Any]    		=False
				lowerCamelCase :			Optional[int]    		=model.config.eos_token_id

				lowerCamelCase :			str    		=jax.jit(model.generate )

				lowerCamelCase :			Optional[int]    		=jit_generate(
				    inputs['''input_ids''']      ,  attention_mask=inputs['''attention_mask''']      ,  pad_token_id=tokenizer.pad_token_id ).sequences

				lowerCamelCase :			Any    		=tokenizer.batch_decode(__lowercase      ,  skip_special_tokens=__lowercase )

				lowerCamelCase :			str    		=[
				    '''Hello this is a long string of text.\n\nI\'m trying to get the text of the''',
				    '''Hey, I\'m a little late to the party. I\'m going to''',
				]

				self.assertListEqual(__lowercase      ,  __lowercase )







		@is_pt_flax_cross_test
		def       __lowercase (		self )					-> int:
				lowerCamelCase		, lowerCamelCase :			int    		=self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						with self.subTest(model_class.__name__ ):
								# prepare inputs
								lowerCamelCase :			Optional[Any]    		=self._prepare_for_class(__lowercase      ,  __lowercase )
								lowerCamelCase :			Optional[Any]    		={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

								# load corresponding PyTorch class
								lowerCamelCase :			Any    		=model_class.__name__[4:]  # Skip the "Flax" at the beginning
								lowerCamelCase :			Optional[int]    		=getattr(__lowercase      ,  __lowercase )

								lowerCamelCase		, lowerCamelCase :			Union[str, Any]    		=pt_inputs['''input_ids'''].shape
								lowerCamelCase :			Dict    		=np.random.randint(0      ,  seq_length - 1      ,  size=(batch_size,) )
								for batch_idx, start_index in enumerate(__lowercase ):
										lowerCamelCase :			Tuple    		=0
										lowerCamelCase :			Optional[Any]    		=1
										lowerCamelCase :			Tuple    		=0
										lowerCamelCase :			Union[str, Any]    		=1
								lowerCamelCase :			Optional[int]    		=pt_model_class(__lowercase ).eval()
								lowerCamelCase :			Any    		=model_class(__lowercase      ,  dtype=jnp.floataa )

								lowerCamelCase :			Union[str, Any]    		=convert_pytorch_state_dict_to_flax(pt_model.state_dict()      ,  __lowercase )
								lowerCamelCase :			Optional[int]    		=fx_state

								with torch.no_grad():
										lowerCamelCase :			List[Any]    		=pt_model(**__lowercase ).to_tuple()

								lowerCamelCase :			List[Any]    		=fx_model(**__lowercase ).to_tuple()
								self.assertEqual(len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )

								with tempfile.TemporaryDirectory() as tmpdirname:
										pt_model.save_pretrained(__lowercase )
										lowerCamelCase :			List[str]    		=model_class.from_pretrained(__lowercase      ,  from_pt=__lowercase )

								lowerCamelCase :			List[str]    		=fx_model_loaded(**__lowercase ).to_tuple()
								self.assertEqual(
								    len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output_loaded, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output_loaded[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )







		@is_pt_flax_cross_test
		def       __lowercase (		self )					-> str:
				lowerCamelCase		, lowerCamelCase :			Dict    		=self.model_tester.prepare_config_and_inputs_for_common()
				for model_class in self.all_model_classes:
						with self.subTest(model_class.__name__ ):
								# prepare inputs
								lowerCamelCase :			Tuple    		=self._prepare_for_class(__lowercase      ,  __lowercase )
								lowerCamelCase :			Any    		={k: torch.tensor(v.tolist() ) for k, v in prepared_inputs_dict.items()}

								# load corresponding PyTorch class
								lowerCamelCase :			Optional[Any]    		=model_class.__name__[4:]  # Skip the "Flax" at the beginning
								lowerCamelCase :			List[str]    		=getattr(__lowercase      ,  __lowercase )

								lowerCamelCase :			List[str]    		=pt_model_class(__lowercase ).eval()
								lowerCamelCase :			Union[str, Any]    		=model_class(__lowercase      ,  dtype=jnp.floataa )

								lowerCamelCase :			Dict    		=load_flax_weights_in_pytorch_model(__lowercase      ,  fx_model.params )
								lowerCamelCase		, lowerCamelCase :			Any    		=pt_inputs['''input_ids'''].shape
								lowerCamelCase :			str    		=np.random.randint(0      ,  seq_length - 1      ,  size=(batch_size,) )
								for batch_idx, start_index in enumerate(__lowercase ):
										lowerCamelCase :			List[str]    		=0
										lowerCamelCase :			Union[str, Any]    		=1
										lowerCamelCase :			Optional[int]    		=0
										lowerCamelCase :			Dict    		=1

								# make sure weights are tied in PyTorch
								pt_model.tie_weights()

								with torch.no_grad():
										lowerCamelCase :			Tuple    		=pt_model(**__lowercase ).to_tuple()

								lowerCamelCase :			Optional[Any]    		=fx_model(**__lowercase ).to_tuple()
								self.assertEqual(len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )

								with tempfile.TemporaryDirectory() as tmpdirname:
										fx_model.save_pretrained(__lowercase )
										lowerCamelCase :			Union[str, Any]    		=pt_model_class.from_pretrained(__lowercase      ,  from_flax=__lowercase )

								with torch.no_grad():
										lowerCamelCase :			str    		=pt_model_loaded(**__lowercase ).to_tuple()

								self.assertEqual(
								    len(__lowercase )      ,  len(__lowercase )      ,  '''Output lengths differ between Flax and PyTorch''' )
								for fx_output, pt_output in zip(__lowercase      ,  __lowercase ):
										self.assert_almost_equals(fx_output[:, -1]      ,  pt_output[:, -1].numpy()      ,  4e-2 )







		@slow
		def       __lowercase (		self )					-> Any:
				for model_class_name in self.all_model_classes:
						lowerCamelCase :			Optional[Any]    		=model_class_name.from_pretrained('''EleutherAI/gpt-neo-125M''' )
						lowerCamelCase :			Union[str, Any]    		=model(np.ones((1, 1) ) )
						self.assertIsNotNone(__lowercase )

