






'''simple docstring'''







import unittest

import requests

from transformers import MODEL_FOR_VISION_2_SEQ_MAPPING, TF_MODEL_FOR_VISION_2_SEQ_MAPPING, is_vision_available
from transformers.pipelines import pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_torch_available():
  from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
  UpperCamelCase_					=				False


if is_vision_available():
  from PIL import Image
else:




  class 					__UpperCAmelCase :

        '''simple docstring'''





        @staticmethod
        def 							lowerCamelCase     (					*_UpperCAmelCase  ,		**_UpperCAmelCase						):
               pass




@is_pipeline_test
@require_vision
class 					__UpperCAmelCase (		unittest.TestCase	):

      '''simple docstring'''





      SCREAMING_SNAKE_CASE				:    Dict				       =					MODEL_FOR_VISION_2_SEQ_MAPPING
      SCREAMING_SNAKE_CASE				:    Dict				       =					TF_MODEL_FOR_VISION_2_SEQ_MAPPING




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   Optional[Any]												=	pipeline('''image-to-text'''  ,		model=_UpperCAmelCase  ,		tokenizer=_UpperCAmelCase  ,		image_processor=_UpperCAmelCase						)
             UpperCAmelCase__	:   Any												=	[
                 Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''						),
                 "./tests/fixtures/tests_samples/COCO/000000039769.png",
             ]
             return pipe, examples




      def 							lowerCamelCase     (					self  ,		_UpperCAmelCase  ,		_UpperCAmelCase						):
             UpperCAmelCase__	:   str												=	pipe(_UpperCAmelCase						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     [{'''generated_text''': ANY(_UpperCAmelCase						)}],
                     [{'''generated_text''': ANY(_UpperCAmelCase						)}],
                 ]  ,		)




      @require_tf
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   str												=	pipeline('''image-to-text'''  ,		model='''hf-internal-testing/tiny-random-vit-gpt2'''  ,		framework='''tf'''						)
             UpperCAmelCase__	:   int												=	"./tests/fixtures/tests_samples/COCO/000000039769.png"

             UpperCAmelCase__	:   Tuple												=	pipe(_UpperCAmelCase						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     {
                         '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                     },
                 ]  ,		)

             UpperCAmelCase__	:   Optional[int]												=	pipe([image, image]						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     [
                         {
                             '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                         }
                     ],
                     [
                         {
                             '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                         }
                     ],
                 ]  ,		)

             UpperCAmelCase__	:   Any												=	pipe(_UpperCAmelCase  ,		max_new_tokens=1						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[{'''generated_text''': '''growth'''}]  ,		)




      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   List[Any]												=	pipeline('''image-to-text'''  ,		model='''hf-internal-testing/tiny-random-vit-gpt2'''						)
             UpperCAmelCase__	:   str												=	"./tests/fixtures/tests_samples/COCO/000000039769.png"

             UpperCAmelCase__	:   str												=	pipe(_UpperCAmelCase						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     {
                         '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                     },
                 ]  ,		)

             UpperCAmelCase__	:   Optional[int]												=	pipe([image, image]						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     [
                         {
                             '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                         }
                     ],
                     [
                         {
                             '''generated_text''': '''growthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthgrowthGOGO'''
                         }
                     ],
                 ]  ,		)




      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   List[Any]												=	pipeline('''image-to-text'''  ,		model='''hf-internal-testing/tiny-random-BlipForConditionalGeneration'''						)
             UpperCAmelCase__	:   Union[str, Any]												=	"./tests/fixtures/tests_samples/COCO/000000039769.png"
             UpperCAmelCase__	:   Optional[Any]												=	"a photo of"

             UpperCAmelCase__	:   Any												=	pipe(_UpperCAmelCase  ,		prompt=_UpperCAmelCase						)
             self.assertTrue(outputs[0]['''generated_text'''].startswith(_UpperCAmelCase						)						)




      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   List[Any]												=	pipeline('''image-to-text'''  ,		model='''ydshieh/vit-gpt2-coco-en'''						)
             UpperCAmelCase__	:   List[str]												=	"./tests/fixtures/tests_samples/COCO/000000039769.png"

             UpperCAmelCase__	:   List[str]												=	pipe(_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}]						)

             UpperCAmelCase__	:   Optional[Any]												=	pipe([image, image]						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     [{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}],
                     [{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}],
                 ]  ,		)




      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Any												=	pipeline('''image-to-text'''  ,		model='''Salesforce/blip-image-captioning-base'''						)
             UpperCAmelCase__	:   str												=	"https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png"
             UpperCAmelCase__	:   Optional[Any]												=	Image.open(requests.get(_UpperCAmelCase  ,		stream=_UpperCAmelCase						).raw						)

             UpperCAmelCase__	:   Any												=	pipe(_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a pink pokemon pokemon with a blue shirt and a blue shirt'''}]						)




      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   str												=	pipeline('''image-to-text'''  ,		model='''microsoft/git-base-coco'''						)
             UpperCAmelCase__	:   Union[str, Any]												=	"https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/pokemon.png"
             UpperCAmelCase__	:   int												=	Image.open(requests.get(_UpperCAmelCase  ,		stream=_UpperCAmelCase						).raw						)

             UpperCAmelCase__	:   Tuple												=	pipe(_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a cartoon of a purple character.'''}]						)




      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Any												=	pipeline('''image-to-text'''  ,		model='''Salesforce/blip-image-captioning-base'''						)
             UpperCAmelCase__	:   Union[str, Any]												=	"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
             UpperCAmelCase__	:   int												=	Image.open(requests.get(_UpperCAmelCase  ,		stream=_UpperCAmelCase						).raw						)

             UpperCAmelCase__	:   Any												=	"a photography of"

             UpperCAmelCase__	:   Optional[Any]												=	pipe(_UpperCAmelCase  ,		prompt=_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a photography of a volcano'''}]						)

             with self.assertRaises(_UpperCAmelCase						):
                    UpperCAmelCase__	:   Tuple												=	pipe([image, image]  ,		prompt=[prompt, prompt]						)




      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[int]												=	pipeline('''image-to-text'''  ,		model='''microsoft/git-base-coco'''						)
             UpperCAmelCase__	:   List[str]												=	"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
             UpperCAmelCase__	:   List[str]												=	Image.open(requests.get(_UpperCAmelCase  ,		stream=_UpperCAmelCase						).raw						)

             UpperCAmelCase__	:   Optional[Any]												=	"a photo of a"

             UpperCAmelCase__	:   Tuple												=	pipe(_UpperCAmelCase  ,		prompt=_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a photo of a tent with a tent and a tent in the background.'''}]						)

             with self.assertRaises(_UpperCAmelCase						):
                    UpperCAmelCase__	:   Tuple												=	pipe([image, image]  ,		prompt=[prompt, prompt]						)




      @unittest.skipIf(
          not is_torch_greater_or_equal_than_1_11  ,		reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''						)
      @slow
      @require_torch
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Optional[int]												=	pipeline('''image-to-text'''  ,		model='''google/pix2struct-ai2d-base'''						)
             UpperCAmelCase__	:   Any												=	"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/ai2d-demo.jpg"
             UpperCAmelCase__	:   Union[str, Any]												=	Image.open(requests.get(_UpperCAmelCase  ,		stream=_UpperCAmelCase						).raw						)

             UpperCAmelCase__	:   List[Any]												=	"What does the label 15 represent? (1) lava (2) core (3) tunnel (4) ash cloud"

             UpperCAmelCase__	:   Dict												=	pipe(_UpperCAmelCase  ,		prompt=_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''ash cloud'''}]						)

             with self.assertRaises(_UpperCAmelCase						):
                    UpperCAmelCase__	:   List[str]												=	pipe([image, image]  ,		prompt=[prompt, prompt]						)




      @slow
      @require_tf
      def 							lowerCamelCase     (					self						):
             UpperCAmelCase__	:   Union[str, Any]												=	pipeline('''image-to-text'''  ,		model='''ydshieh/vit-gpt2-coco-en'''  ,		framework='''tf'''						)
             UpperCAmelCase__	:   str												=	"./tests/fixtures/tests_samples/COCO/000000039769.png"

             UpperCAmelCase__	:   Union[str, Any]												=	pipe(_UpperCAmelCase						)
             self.assertEqual(_UpperCAmelCase  ,		[{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}]						)

             UpperCAmelCase__	:   int												=	pipe([image, image]						)
             self.assertEqual(
                 _UpperCAmelCase  ,		[
                     [{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}],
                     [{'''generated_text''': '''a cat laying on a blanket next to a cat laying on a bed '''}],
                 ]  ,		)




import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments

import datasets


__lowerCamelCase								=      '''\
@article{eddine2021frugalscore,
  title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation},
  author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis},
  journal={arXiv preprint arXiv:2110.08559},
  year={2021}
}
'''

__lowerCamelCase								=      '''\
FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance.
'''


__lowerCamelCase								=      '''
Calculates how good are predictions given some references, using certain scores.
Args:
    predictions (list of str): list of predictions to score. Each predictions
        should be a string.
    references (list of str): list of reference for each prediction. Each
        reference should be a string.
    batch_size (int): the batch size for predictions.
    max_length (int): maximum sequence length.
    device (str): either gpu or cpu
Returns:
    scores (list of int): list of scores.
Examples:
    >>> frugalscore = datasets.load_metric("frugalscore")
    >>> results = frugalscore.compute(predictions=[\'hello there\', \'huggingface\'], references=[\'hello world\', \'hugging face\'])
    >>> print([round(s, 3) for s in results["scores"]])
    [0.631, 0.645]
'''


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION			,			_KWARGS_DESCRIPTION	)
class 	snake_case_			(datasets.Metric	):




       """simple docstring"""






       def 			A_ (  self):

             """simple docstring"""





             return datasets.MetricInfo(
                 description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(
                     {
                         "predictions": datasets.Value("string"),
                         "references": datasets.Value("string"),
                     })  ,homepage="https://github.com/moussaKam/FrugalScore"  ,)



       def 			A_ (  self  ,lowercase):

             """simple docstring"""





             if self.config_name == "default":
                   UpperCAmelCase_	:						str					   =		"moussaKam/frugalscore_tiny_bert-base_bert-score"
             else:
                   UpperCAmelCase_	:						Optional[int]					   =		self.config_name
             UpperCAmelCase_	:						Dict					   =		AutoModelForSequenceClassification.from_pretrained(lowercase)
             UpperCAmelCase_	:						str					   =		AutoTokenizer.from_pretrained(lowercase)



       def 			A_ (  self  ,lowercase  ,lowercase  ,lowercase=32  ,lowercase=128  ,lowercase=None  ,):

             """simple docstring"""





             assert len(lowercase) == len(
                 lowercase), "predictions and references should have the same number of sentences."
             if device is not None:
                   assert device in ["gpu", "cpu"], "device should be either gpu or cpu."
             else:
                   UpperCAmelCase_	:						Union[str, Any]					   =		"gpu" if torch.cuda.is_available() else "cpu"
             UpperCAmelCase_	:						Optional[Any]					   =		TrainingArguments(
                 "trainer"  ,fpaa=(device == "gpu")  ,per_device_eval_batch_size=lowercase  ,report_to="all"  ,no_cuda=(device == "cpu")  ,log_level="warning"  ,)
             UpperCAmelCase_	:						List[str]					   =		{"sentence1": predictions, "sentence2": references}
             UpperCAmelCase_	:						List[Any]					   =		datasets.Dataset.from_dict(lowercase)

             def tokenize_function(lowercase):
                   return self.tokenizer(
                       data["sentence1"]  ,data["sentence2"]  ,max_length=lowercase  ,truncation=lowercase  ,padding=lowercase)

             UpperCAmelCase_	:						int					   =		raw_datasets.map(lowercase  ,batched=lowercase)
             tokenized_datasets.remove_columns(["sentence1", "sentence2"])
             UpperCAmelCase_	:						Union[str, Any]					   =		Trainer(self.model  ,lowercase  ,tokenizer=self.tokenizer)
             UpperCAmelCase_	:						Tuple					   =		trainer.predict(lowercase)
             return {"scores": list(predictions.predictions.squeeze(-1))}
