





from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
    from fairscale.optim import OSS


__lowerCamelCase       :     Dict   =     logging.get_logger(__name__)

__lowerCamelCase       :     Tuple   =     {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}



class 					A__   (       __snake_case			):





      def __init__( self       ,							A_=None       ,							A_=None       ,							*A_       ,							**A_   ):
       '''simple docstring'''




       super().__init__(*A_       ,							**A_   )

       if config is None:
        assert isinstance(self.model       ,							A_   ), (
            "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
            F""" {self.model.__class__}"""
        )
        UpperCamelCase	:					Optional[Any]       	=   self.model.config
       else:
        UpperCamelCase	:					int       	=   config

       UpperCamelCase	:					Tuple       	=   data_args
       UpperCamelCase	:					List[str]       	=   self.config.tgt_vocab_size if isinstance(self.config       ,							A_   ) else self.config.vocab_size

       if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
        assert self.config.pad_token_id is not None, (
            "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
            " calculation or doing label smoothing."
        )

       if self.config.pad_token_id is None and self.config.eos_token_id is not None:
        logger.warning(
            F"""The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for"""
            " padding.."   )

       if self.args.label_smoothing == 0:
        UpperCamelCase	:					Any       	=   torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id   )
       else:
        # dynamically import label_smoothed_nll_loss
        from utils import label_smoothed_nll_loss

        UpperCamelCase	:					Optional[Any]       	=   label_smoothed_nll_loss





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       if self.optimizer is None:
        UpperCamelCase	:					int       	=   ["bias", "LayerNorm.weight"]
        UpperCamelCase	:					Any       	=   [
            {
                "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay   )],
                "weight_decay": self.args.weight_decay,
            },
            {
                "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay   )],
                "weight_decay": 0.0,
            },
        ]
        UpperCamelCase	:					Optional[int]       	=   Adafactor if self.args.adafactor else AdamW
        if self.args.adafactor:
         UpperCamelCase	:					Dict       	=   Adafactor
         UpperCamelCase	:					List[Any]       	=   {"scale_parameter": False, "relative_step": False}
        else:
         UpperCamelCase	:					Optional[int]       	=   AdamW
         UpperCamelCase	:					List[Any]       	=   {
             "betas": (self.args.adam_betaa, self.args.adam_betaa),
             "eps": self.args.adam_epsilon,
         }
        UpperCamelCase	:					Optional[int]       	=   self.args.learning_rate
        if self.sharded_ddp:
         UpperCamelCase	:					Dict       	=   OSS(
             params=A_       ,							optim=A_       ,							**A_       ,							)
        else:
         UpperCamelCase	:					Dict       	=   optimizer_cls(A_       ,							**A_   )

       if self.lr_scheduler is None:
        UpperCamelCase	:					int       	=   self._get_lr_scheduler(A_   )
       else:  # ignoring --lr_scheduler
        logger.warning("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."   )





      def        __UpperCamelCase( self       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					List[str]       	=   arg_to_scheduler[self.args.lr_scheduler]
       if self.args.lr_scheduler == "constant":
        UpperCamelCase	:					Any       	=   schedule_func(self.optimizer   )
       elif self.args.lr_scheduler == "constant_w_warmup":
        UpperCamelCase	:					List[Any]       	=   schedule_func(self.optimizer       ,							num_warmup_steps=self.args.warmup_steps   )
       else:
        UpperCamelCase	:					str       	=   schedule_func(
            self.optimizer       ,							num_warmup_steps=self.args.warmup_steps       ,							num_training_steps=A_   )
       return scheduler





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       if isinstance(self.train_dataset       ,							torch.utils.data.IterableDataset   ):
        return None
       elif is_torch_tpu_available():
        return get_tpu_sampler(self.train_dataset   )
       else:
        if self.args.sortish_sampler:
         self.train_dataset.make_sortish_sampler(
             self.args.per_device_train_batch_size       ,							distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)       ,							)

        return (
            RandomSampler(self.train_dataset   )
            if self.args.local_rank == -1
            else DistributedSampler(self.train_dataset   )
        )





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_   ):
       '''simple docstring'''




       if self.args.label_smoothing == 0:
        if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
         # force training to ignore pad token
         UpperCamelCase	:					List[Any]       	=   model(**A_       ,							use_cache=A_   )[0]
         UpperCamelCase	:					Union[str, Any]       	=   self.loss_fn(logits.view(-1       ,							logits.shape[-1]   )       ,							labels.view(-1   )   )
        else:
         # compute usual loss via models
         UpperCamelCase	,    UpperCamelCase	:					str       	=   model(**A_       ,							labels=A_       ,							use_cache=A_   )[:2]
       else:
        # compute label smoothed loss
        UpperCamelCase	:					Tuple       	=   model(**A_       ,							use_cache=A_   )[0]
        UpperCamelCase	:					List[str]       	=   torch.nn.functional.log_softmax(A_       ,							dim=-1   )
        UpperCamelCase	,    UpperCamelCase	:					Dict       	=   self.loss_fn(A_       ,							A_       ,							self.args.label_smoothing       ,							ignore_index=self.config.pad_token_id   )
       return loss, logits





      def        __UpperCamelCase( self       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Dict       	=   inputs.pop("labels"   )
       UpperCamelCase	,    UpperCamelCase	:					int       	=   self._compute_loss(A_       ,							A_       ,							A_   )
       return loss





      def        __UpperCamelCase( self       ,							A_       ,							A_       ,							A_       ,							A_ = None       ,							):
       '''simple docstring'''




       UpperCamelCase	:					str       	=   self._prepare_inputs(A_   )

       UpperCamelCase	:					Optional[Any]       	=   {
           "max_length": self.data_args.val_max_target_length
           if self.data_args is not None
           else self.config.max_length,
           "num_beams": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
       }

       if self.args.predict_with_generate and not self.args.prediction_loss_only:
        UpperCamelCase	:					Optional[Any]       	=   self.model.generate(
            inputs["input_ids"]       ,							attention_mask=inputs["attention_mask"]       ,							**A_       ,							)
        # in case the batch is shorter than max length, the output should be padded
        if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
         UpperCamelCase	:					Tuple       	=   self._pad_tensors_to_max_len(A_       ,							gen_kwargs["max_length"]   )

       UpperCamelCase	:					List[str]       	=   inputs.pop("labels"   )
       with torch.no_grad():
        # compute loss on predict data
        UpperCamelCase	,    UpperCamelCase	:					str       	=   self._compute_loss(A_       ,							A_       ,							A_   )

       UpperCamelCase	:					Dict       	=   loss.mean().detach()
       if self.args.prediction_loss_only:
        return (loss, None, None)

       UpperCamelCase	:					Optional[int]       	=   generated_tokens if self.args.predict_with_generate else logits

       if labels.shape[-1] < gen_kwargs["max_length"]:
        UpperCamelCase	:					Dict       	=   self._pad_tensors_to_max_len(A_       ,							gen_kwargs["max_length"]   )

       return (loss, logits, labels)





      def        __UpperCamelCase( self       ,							A_       ,							A_   ):
       '''simple docstring'''




       UpperCamelCase	:					Any       	=   self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

       if pad_token_id is None:
        raise ValueError(
            "Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"
            F""" padded to `max_length`={max_length}"""   )

       UpperCamelCase	:					List[Any]       	=   pad_token_id * torch.ones(
           (tensor.shape[0], max_length)       ,							dtype=tensor.dtype       ,							device=tensor.device   )
       UpperCamelCase	:					Dict       	=   tensor
       return padded_tensor







import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__lowerCamelCase       :     Union[str, Any]   =     pytest.mark.integration



@require_faiss
class 					A__   (       __snake_case			):





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Union[str, Any]       	=   Dataset.from_dict({"filename": ["my_name-train" + "_" + str(A_   ) for x in np.arange(30   ).tolist()]}   )
       return dset





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					Dataset       	=   self._create_dummy_dataset()
       UpperCamelCase	:					List[Any]       	=   dset.map(
           lambda A_       ,							A_   : {"vecs": i * np.ones(5       ,							dtype=np.floataa   )}       ,							with_indices=A_       ,							keep_in_memory=A_   )
       UpperCamelCase	:					List[str]       	=   dset.add_faiss_index("vecs"       ,							batch_size=100       ,							metric_type=faiss.METRIC_INNER_PRODUCT   )
       UpperCamelCase	,    UpperCamelCase	:					Tuple       	=   dset.get_nearest_examples("vecs"       ,							np.ones(5       ,							dtype=np.floataa   )   )
       self.assertEqual(examples["filename"][0]       ,							"my_name-train_29"   )
       dset.drop_index("vecs"   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					Dataset       	=   self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)   ) * np.arange(30   ).reshape(-1       ,							1   )       ,							index_name="vecs"       ,							batch_size=100       ,							metric_type=faiss.METRIC_INNER_PRODUCT       ,							)
       UpperCamelCase	,    UpperCamelCase	:					int       	=   dset.get_nearest_examples("vecs"       ,							np.ones(5       ,							dtype=np.floataa   )   )
       self.assertEqual(examples["filename"][0]       ,							"my_name-train_29"   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					Dataset       	=   self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)   ) * np.arange(30   ).reshape(-1       ,							1   )       ,							index_name="vecs"       ,							metric_type=faiss.METRIC_INNER_PRODUCT       ,							)

       # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
       # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
       # see https://bugs.python.org/issue14243 and
       # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
       with tempfile.NamedTemporaryFile(delete=A_   ) as tmp_file:
        dset.save_faiss_index("vecs"       ,							tmp_file.name   )
        dset.load_faiss_index("vecs2"       ,							tmp_file.name   )
       os.unlink(tmp_file.name   )

       UpperCamelCase	,    UpperCamelCase	:					List[str]       	=   dset.get_nearest_examples("vecs2"       ,							np.ones(5       ,							dtype=np.floataa   )   )
       self.assertEqual(examples["filename"][0]       ,							"my_name-train_29"   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       UpperCamelCase	:					Dataset       	=   self._create_dummy_dataset()
       dset.add_faiss_index_from_external_arrays(
           external_arrays=np.ones((30, 5)   ) * np.arange(30   ).reshape(-1       ,							1   )       ,							index_name="vecs"   )
       dset.drop_index("vecs"   )
       self.assertRaises(A_       ,							partial(dset.get_nearest_examples       ,							"vecs2"       ,							np.ones(5       ,							dtype=np.floataa   )   )   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       from elasticsearch import Elasticsearch

       UpperCamelCase	:					Dataset       	=   self._create_dummy_dataset()
       with patch("elasticsearch.Elasticsearch.search"   ) as mocked_search, patch(
           "elasticsearch.client.IndicesClient.create"   ) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"   ) as mocked_bulk:
        UpperCamelCase	:					List[str]       	=   {"acknowledged": True}
        mocked_bulk.return_value([(True, None)] * 30   )
        UpperCamelCase	:					List[Any]       	=   {"hits": {"hits": [{"_score": 1, "_id": 29}]}}
        UpperCamelCase	:					Optional[Any]       	=   Elasticsearch()

        dset.add_elasticsearch_index("filename"       ,							es_client=A_   )
        UpperCamelCase	,    UpperCamelCase	:					List[str]       	=   dset.get_nearest_examples("filename"       ,							"my_name-train_29"   )
        self.assertEqual(examples["filename"][0]       ,							"my_name-train_29"   )



@require_faiss
class 					A__   (       __snake_case			):





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					Optional[int]       	=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT   )

       # add vectors
       index.add_vectors(np.eye(5       ,							dtype=np.floataa   )   )
       self.assertIsNotNone(index.faiss_index   )
       self.assertEqual(index.faiss_index.ntotal       ,							5   )
       index.add_vectors(np.zeros((5, 5)       ,							dtype=np.floataa   )   )
       self.assertEqual(index.faiss_index.ntotal       ,							10   )

       # single query
       UpperCamelCase	:					Any       	=   np.zeros(5       ,							dtype=np.floataa   )
       UpperCamelCase	:					Optional[Any]       	=   1
       UpperCamelCase	,    UpperCamelCase	:					Optional[Any]       	=   index.search(A_   )
       self.assertRaises(A_       ,							index.search       ,							query.reshape(-1       ,							1   )   )
       self.assertGreater(scores[0]       ,							0   )
       self.assertEqual(indices[0]       ,							1   )

       # batched queries
       UpperCamelCase	:					Optional[int]       	=   np.eye(5       ,							dtype=np.floataa   )[::-1]
       UpperCamelCase	,    UpperCamelCase	:					Tuple       	=   index.search_batch(A_   )
       self.assertRaises(A_       ,							index.search_batch       ,							queries[0]   )
       UpperCamelCase	:					Optional[int]       	=   [scores[0] for scores in total_scores]
       UpperCamelCase	:					Tuple       	=   [indices[0] for indices in total_indices]
       self.assertGreater(np.min(A_   )       ,							0   )
       self.assertListEqual([4, 3, 2, 1, 0]       ,							A_   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					List[str]       	=   FaissIndex(string_factory="Flat"   )
       index.add_vectors(np.eye(5       ,							dtype=np.floataa   )   )
       self.assertIsInstance(index.faiss_index       ,							faiss.IndexFlat   )
       UpperCamelCase	:					List[str]       	=   FaissIndex(string_factory="LSH"   )
       index.add_vectors(np.eye(5       ,							dtype=np.floataa   )   )
       self.assertIsInstance(index.faiss_index       ,							faiss.IndexLSH   )
       with self.assertRaises(A_   ):
        UpperCamelCase	:					List[str]       	=   FaissIndex(string_factory="Flat"       ,							custom_index=faiss.IndexFlat(5   )   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					Dict       	=   faiss.IndexFlat(5   )
       UpperCamelCase	:					Union[str, Any]       	=   FaissIndex(custom_index=A_   )
       index.add_vectors(np.eye(5       ,							dtype=np.floataa   )   )
       self.assertIsInstance(index.faiss_index       ,							faiss.IndexFlat   )





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       import faiss

       UpperCamelCase	:					str       	=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT   )
       index.add_vectors(np.eye(5       ,							dtype=np.floataa   )   )

       # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
       # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
       # see https://bugs.python.org/issue14243 and
       # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
       with tempfile.NamedTemporaryFile(delete=A_   ) as tmp_file:
        index.save(tmp_file.name   )
        UpperCamelCase	:					int       	=   FaissIndex.load(tmp_file.name   )
       os.unlink(tmp_file.name   )

       UpperCamelCase	:					str       	=   np.zeros(5       ,							dtype=np.floataa   )
       UpperCamelCase	:					int       	=   1
       UpperCamelCase	,    UpperCamelCase	:					Dict       	=   index.search(A_   )
       self.assertGreater(scores[0]       ,							0   )
       self.assertEqual(indices[0]       ,							1   )



@require_faiss
def 		A_				(     _lowerCAmelCase     )						-> Optional[int]:
 import faiss

 UpperCamelCase	:					Union[str, Any]       	=   FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
 index.add_vectors(np.eye(5							,   dtype=np.floataa     )     )

 UpperCamelCase	:					List[Any]       	=   "index.faiss"
 UpperCamelCase	:					List[str]       	=   F"""mock://{index_name}"""
 index.save(_lowerCAmelCase							,   storage_options=mockfs.storage_options     )
 UpperCamelCase	:					List[str]       	=   FaissIndex.load(_lowerCAmelCase							,   storage_options=mockfs.storage_options     )

 UpperCamelCase	:					List[str]       	=   np.zeros(5							,   dtype=np.floataa     )
 UpperCamelCase	:					Optional[int]       	=   1
 UpperCamelCase	,    UpperCamelCase	:					List[str]       	=   index.search(_lowerCAmelCase     )
 assert scores[0] > 0
 assert indices[0] == 1



@require_elasticsearch
class 					A__   (       __snake_case			):





      def        __UpperCamelCase( self   ):
       '''simple docstring'''




       from elasticsearch import Elasticsearch

       with patch("elasticsearch.Elasticsearch.search"   ) as mocked_search, patch(
           "elasticsearch.client.IndicesClient.create"   ) as mocked_index_create, patch("elasticsearch.helpers.streaming_bulk"   ) as mocked_bulk:
        UpperCamelCase	:					List[str]       	=   Elasticsearch()
        UpperCamelCase	:					Union[str, Any]       	=   {"acknowledged": True}
        UpperCamelCase	:					Union[str, Any]       	=   ElasticSearchIndex(es_client=A_   )
        mocked_bulk.return_value([(True, None)] * 3   )
        index.add_documents(["foo", "bar", "foobar"]   )

        # single query
        UpperCamelCase	:					str       	=   "foo"
        UpperCamelCase	:					Dict       	=   {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
        UpperCamelCase	,    UpperCamelCase	:					Tuple       	=   index.search(A_   )
        self.assertEqual(scores[0]       ,							1   )
        self.assertEqual(indices[0]       ,							0   )

        # single query with timeout
        UpperCamelCase	:					Dict       	=   "foo"
        UpperCamelCase	:					Optional[Any]       	=   {"hits": {"hits": [{"_score": 1, "_id": 0}]}}
        UpperCamelCase	,    UpperCamelCase	:					str       	=   index.search(A_       ,							request_timeout=30   )
        self.assertEqual(scores[0]       ,							1   )
        self.assertEqual(indices[0]       ,							0   )

        # batched queries
        UpperCamelCase	:					Dict       	=   ["foo", "bar", "foobar"]
        UpperCamelCase	:					List[Any]       	=   {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
        UpperCamelCase	,    UpperCamelCase	:					Optional[int]       	=   index.search_batch(A_   )
        UpperCamelCase	:					str       	=   [scores[0] for scores in total_scores]
        UpperCamelCase	:					Optional[Any]       	=   [indices[0] for indices in total_indices]
        self.assertGreater(np.min(A_   )       ,							0   )
        self.assertListEqual([1, 1, 1]       ,							A_   )

        # batched queries with timeout
        UpperCamelCase	:					int       	=   ["foo", "bar", "foobar"]
        UpperCamelCase	:					List[Any]       	=   {"hits": {"hits": [{"_score": 1, "_id": 1}]}}
        UpperCamelCase	,    UpperCamelCase	:					Union[str, Any]       	=   index.search_batch(A_       ,							request_timeout=30   )
        UpperCamelCase	:					Union[str, Any]       	=   [scores[0] for scores in total_scores]
        UpperCamelCase	:					Dict       	=   [indices[0] for indices in total_indices]
        self.assertGreater(np.min(A_   )       ,							0   )
        self.assertListEqual([1, 1, 1]       ,							A_   )
