import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
   import torch

   from ..models.auto.modeling_auto import (
       MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
       MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
   )

if is_tf_available() and is_tensorflow_probability_available():
   import tensorflow as tf
   import tensorflow_probability as tfp

   from ..models.auto.modeling_tf_auto import (
       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
       TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
   )







class __snake_case							(		lowerCamelCase_						):




   def __call__(       self     :							Dict			,      _lowercase     :							str=None			,      _lowercase     :							Optional[Any]=None			,      **_lowercase     :							List[str] ):



     """simple docstring"""






     requires_backends(self			,      """pandas""" )
     import pandas as pd

     if table is None:
       raise ValueError("""Keyword argument `table` cannot be None.""" )
     elif query is None:
       if isinstance(_lowercase			,      _lowercase ) and table.get("""query""" ) is not None and table.get("""table""" ) is not None:
         SCREAMING_SNAKE_CASE__     				= [table]
       elif isinstance(_lowercase			,      _lowercase ) and len(_lowercase ) > 0:
         if not all(isinstance(_lowercase			,      _lowercase ) for d in table ):
           raise ValueError(
               f"""Keyword argument `table` should be a list of dict, but is {(type(_lowercase ) for d in table)}""" )

         if table[0].get("""query""" ) is not None and table[0].get("""table""" ) is not None:
           SCREAMING_SNAKE_CASE__     				= table
         else:
           raise ValueError(
               """If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"""
               f""" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.""" )
       elif Dataset is not None and isinstance(_lowercase			,      _lowercase ) or isinstance(_lowercase			,      types.GeneratorType ):
         return table
       else:
         raise ValueError(
             """Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but """
             f"""is {type(_lowercase )})""" )
     else:
       SCREAMING_SNAKE_CASE__     				= [{"""table""": table, """query""": query}]

     for tqa_pipeline_input in tqa_pipeline_inputs:
       if not isinstance(tqa_pipeline_input["""table"""]			,      pd.DataFrame ):
         if tqa_pipeline_input["table"] is None:
           raise ValueError("""Table cannot be None.""" )

         SCREAMING_SNAKE_CASE__     				= pd.DataFrame(tqa_pipeline_input["""table"""] )

     return tqa_pipeline_inputs







@add_end_docstrings(lowerCamelCase_						)
class __snake_case							(		lowerCamelCase_						):
   lowerCAmelCase_								=				"table,query"




   def __init__(       self     :							Any			,      _lowercase     :							Optional[int]=TableQuestionAnsweringArgumentHandler()			,      *_lowercase     :							int			,      **_lowercase     :							Any ):



     """simple docstring"""






     super().__init__(*_lowercase			,      **_lowercase )
     SCREAMING_SNAKE_CASE__     				= args_parser

     self.check_model_type(
         dict(
             TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
             + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items() )
         if self.framework == """tf"""
         else dict(
             MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items() ) )

     SCREAMING_SNAKE_CASE__     				= bool(getattr(self.model.config			,      """aggregation_labels"""			,      _lowercase ) ) and bool(
         getattr(self.model.config			,      """num_aggregation_labels"""			,      _lowercase ) )
     SCREAMING_SNAKE_CASE__     				= """tapas""" if hasattr(self.model.config			,      """aggregation_labels""" ) else None




   def       __a		(       self     :							Union[str, Any]			,      **_lowercase     :							int ):



     """simple docstring"""






     return self.model(**_lowercase )




   def       __a		(       self     :							Optional[Any]			,      **_lowercase     :							Tuple ):



     """simple docstring"""






     if self.framework == "pt":
       SCREAMING_SNAKE_CASE__     				= []
       SCREAMING_SNAKE_CASE__     				= []
       SCREAMING_SNAKE_CASE__     				= None
       SCREAMING_SNAKE_CASE__     				= inputs["""input_ids"""].shape[0]

       SCREAMING_SNAKE_CASE__     				= inputs["""input_ids"""].to(self.device )
       SCREAMING_SNAKE_CASE__     				= inputs["""attention_mask"""].to(self.device )
       SCREAMING_SNAKE_CASE__     				= inputs["""token_type_ids"""].to(self.device )
       SCREAMING_SNAKE_CASE__     				= None

       for index in range(_lowercase ):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 3]  # shape (seq_len,)
           SCREAMING_SNAKE_CASE__     				= np.zeros_like(prev_labels_example.cpu().numpy() )  # shape (seq_len,)

           SCREAMING_SNAKE_CASE__     				= token_type_ids[index]  # shape (seq_len, 7)
           for i in range(model_labels.shape[0] ):
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 0].tolist()[i]
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 1].tolist()[i] - 1
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 2].tolist()[i] - 1

             if row_id >= 0 and col_id >= 0 and segment_id == 1:
               SCREAMING_SNAKE_CASE__     				= int(prev_answers[(col_id, row_id)] )

           SCREAMING_SNAKE_CASE__     				= torch.from_numpy(_lowercase ).type(torch.long ).to(self.device )

         SCREAMING_SNAKE_CASE__     				= input_ids[index]
         SCREAMING_SNAKE_CASE__     				= attention_mask[index]  # shape (seq_len,)
         SCREAMING_SNAKE_CASE__     				= token_type_ids[index]  # shape (seq_len, 7)
         SCREAMING_SNAKE_CASE__     				= self.model(
             input_ids=input_ids_example.unsqueeze(0 )			,      attention_mask=attention_mask_example.unsqueeze(0 )			,      token_type_ids=token_type_ids_example.unsqueeze(0 )			,      )
         SCREAMING_SNAKE_CASE__     				= outputs.logits

         if self.aggregate:
           all_aggregations.append(outputs.logits_aggregation )

         all_logits.append(_lowercase )

         SCREAMING_SNAKE_CASE__     				= torch.distributions.Bernoulli(logits=_lowercase )
         SCREAMING_SNAKE_CASE__     				= dist_per_token.probs * attention_mask_example.type(torch.floataa ).to(
             dist_per_token.probs.device )

         SCREAMING_SNAKE_CASE__     				= collections.defaultdict(_lowercase )
         for i, p in enumerate(probabilities.squeeze().tolist() ):
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 0].tolist()[i]
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 1].tolist()[i] - 1
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 2].tolist()[i] - 1
           if col >= 0 and row >= 0 and segment_id == 1:
             coords_to_probs[(col, row)].append(_lowercase )

         SCREAMING_SNAKE_CASE__     				= {key: np.array(coords_to_probs[key] ).mean() > 0.5 for key in coords_to_probs}

       SCREAMING_SNAKE_CASE__     				= torch.cat(tuple(_lowercase )			,      0 )

       return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(_lowercase )			,      0 ))
     else:
       SCREAMING_SNAKE_CASE__     				= []
       SCREAMING_SNAKE_CASE__     				= []
       SCREAMING_SNAKE_CASE__     				= None
       SCREAMING_SNAKE_CASE__     				= inputs["""input_ids"""].shape[0]

       SCREAMING_SNAKE_CASE__     				= inputs["""input_ids"""]
       SCREAMING_SNAKE_CASE__     				= inputs["""attention_mask"""]
       SCREAMING_SNAKE_CASE__     				= inputs["""token_type_ids"""].numpy()
       SCREAMING_SNAKE_CASE__     				= None

       for index in range(_lowercase ):
         # If sequences have already been processed, the token type IDs will be created according to the previous
         # answer.
         if prev_answers is not None:
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 3]  # shape (seq_len,)
           SCREAMING_SNAKE_CASE__     				= np.zeros_like(_lowercase			,      dtype=np.intaa )  # shape (seq_len,)

           SCREAMING_SNAKE_CASE__     				= token_type_ids[index]  # shape (seq_len, 7)
           for i in range(model_labels.shape[0] ):
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 0].tolist()[i]
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 1].tolist()[i] - 1
             SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 2].tolist()[i] - 1

             if row_id >= 0 and col_id >= 0 and segment_id == 1:
               SCREAMING_SNAKE_CASE__     				= int(prev_answers[(col_id, row_id)] )

           SCREAMING_SNAKE_CASE__     				= model_labels

         SCREAMING_SNAKE_CASE__     				= input_ids[index]
         SCREAMING_SNAKE_CASE__     				= attention_mask[index]  # shape (seq_len,)
         SCREAMING_SNAKE_CASE__     				= token_type_ids[index]  # shape (seq_len, 7)
         SCREAMING_SNAKE_CASE__     				= self.model(
             input_ids=np.expand_dims(_lowercase			,      axis=0 )			,      attention_mask=np.expand_dims(_lowercase			,      axis=0 )			,      token_type_ids=np.expand_dims(_lowercase			,      axis=0 )			,      )
         SCREAMING_SNAKE_CASE__     				= outputs.logits

         if self.aggregate:
           all_aggregations.append(outputs.logits_aggregation )

         all_logits.append(_lowercase )

         SCREAMING_SNAKE_CASE__     				= tfp.distributions.Bernoulli(logits=_lowercase )
         SCREAMING_SNAKE_CASE__     				= dist_per_token.probs_parameter() * tf.cast(_lowercase			,      tf.floataa )

         SCREAMING_SNAKE_CASE__     				= collections.defaultdict(_lowercase )
         SCREAMING_SNAKE_CASE__     				= token_type_ids_example
         for i, p in enumerate(tf.squeeze(_lowercase ).numpy().tolist() ):
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 0].tolist()[i]
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 1].tolist()[i] - 1
           SCREAMING_SNAKE_CASE__     				= token_type_ids_example[:, 2].tolist()[i] - 1
           if col >= 0 and row >= 0 and segment_id == 1:
             coords_to_probs[(col, row)].append(_lowercase )

         SCREAMING_SNAKE_CASE__     				= {key: np.array(coords_to_probs[key] ).mean() > 0.5 for key in coords_to_probs}

       SCREAMING_SNAKE_CASE__     				= tf.concat(tuple(_lowercase )			,      0 )

       return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(_lowercase )			,      0 ))




   def __call__(       self     :							Optional[int]			,      *_lowercase     :							Union[str, Any]			,      **_lowercase     :							Any ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= self._args_parser(*_lowercase			,      **_lowercase )

     SCREAMING_SNAKE_CASE__     				= super().__call__(_lowercase			,      **_lowercase )
     if len(_lowercase ) == 1:
       return results[0]
     return results




   def       __a		(       self     :							str			,      _lowercase     :							List[str]=None			,      _lowercase     :							List[Any]=None			,      _lowercase     :							Any=None			,      **_lowercase     :							Optional[int] ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= {}
     if padding is not None:
       SCREAMING_SNAKE_CASE__     				= padding
     if truncation is not None:
       SCREAMING_SNAKE_CASE__     				= truncation

     SCREAMING_SNAKE_CASE__     				= {}
     if sequential is not None:
       SCREAMING_SNAKE_CASE__     				= sequential
     return preprocess_params, forward_params, {}




   def       __a		(       self     :							Union[str, Any]			,      _lowercase     :							Optional[Any]			,      _lowercase     :							Any=None			,      _lowercase     :							Optional[Any]=True			,      _lowercase     :							Optional[int]=None ):



     """simple docstring"""






     if truncation is None:
       if self.type == "tapas":
         SCREAMING_SNAKE_CASE__     				= """drop_rows_to_fit"""
       else:
         SCREAMING_SNAKE_CASE__     				= """do_not_truncate"""

     SCREAMING_SNAKE_CASE__				,	SCREAMING_SNAKE_CASE__     				= pipeline_input["""table"""], pipeline_input["""query"""]
     if table.empty:
       raise ValueError("""table is empty""" )
     if query is None or query == "":
       raise ValueError("""query is empty""" )
     SCREAMING_SNAKE_CASE__     				= self.tokenizer(_lowercase			,      _lowercase			,      return_tensors=self.framework			,      truncation=_lowercase			,      padding=_lowercase )
     SCREAMING_SNAKE_CASE__     				= table
     return inputs




   def       __a		(       self     :							Tuple			,      _lowercase     :							List[Any]			,      _lowercase     :							List[str]=False ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= model_inputs.pop("""table""" )

     if self.type == "tapas":
       if sequential:
         SCREAMING_SNAKE_CASE__     				= self.sequential_inference(**_lowercase )
       else:
         SCREAMING_SNAKE_CASE__     				= self.batch_inference(**_lowercase )
     else:
       SCREAMING_SNAKE_CASE__     				= self.model.generate(**_lowercase )
     SCREAMING_SNAKE_CASE__     				= {"""model_inputs""": model_inputs, """table""": table, """outputs""": outputs}
     return model_outputs




   def       __a		(       self     :							int			,      _lowercase     :							Tuple ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__     				= model_outputs["""model_inputs"""]
     SCREAMING_SNAKE_CASE__     				= model_outputs["""table"""]
     SCREAMING_SNAKE_CASE__     				= model_outputs["""outputs"""]
     if self.type == "tapas":
       if self.aggregate:
         SCREAMING_SNAKE_CASE__				,	SCREAMING_SNAKE_CASE__     				= outputs[:2]
         SCREAMING_SNAKE_CASE__     				= self.tokenizer.convert_logits_to_predictions(_lowercase			,      _lowercase			,      _lowercase )
         SCREAMING_SNAKE_CASE__				,	SCREAMING_SNAKE_CASE__     				= predictions
         SCREAMING_SNAKE_CASE__     				= {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(_lowercase )}

         SCREAMING_SNAKE_CASE__     				= self.model.config.no_aggregation_label_index
         SCREAMING_SNAKE_CASE__     				= {
             i: aggregators[i] + """ > """ for i, pred in enumerate(_lowercase ) if pred != no_agg_label_index
         }
       else:
         SCREAMING_SNAKE_CASE__     				= outputs[0]
         SCREAMING_SNAKE_CASE__     				= self.tokenizer.convert_logits_to_predictions(_lowercase			,      _lowercase )
         SCREAMING_SNAKE_CASE__     				= predictions[0]
         SCREAMING_SNAKE_CASE__     				= {}
         SCREAMING_SNAKE_CASE__     				= {}
       SCREAMING_SNAKE_CASE__     				= []
       for index, coordinates in enumerate(_lowercase ):
         SCREAMING_SNAKE_CASE__     				= [table.iat[coordinate] for coordinate in coordinates]
         SCREAMING_SNAKE_CASE__     				= aggregators.get(_lowercase			,      """""" )
         SCREAMING_SNAKE_CASE__     				= aggregators_prefix.get(_lowercase			,      """""" )
         SCREAMING_SNAKE_CASE__     				= {
             """answer""": aggregator_prefix + """, """.join(_lowercase ),
             """coordinates""": coordinates,
             """cells""": [table.iat[coordinate] for coordinate in coordinates],
         }
         if aggregator:
           SCREAMING_SNAKE_CASE__     				= aggregator

         answers.append(_lowercase )
       if len(_lowercase ) == 0:
         raise PipelineException("""Empty answer""" )
     else:
       SCREAMING_SNAKE_CASE__     				= [{"""answer""": answer} for answer in self.tokenizer.batch_decode(_lowercase			,      skip_special_tokens=_lowercase )]

     return answers if len(_lowercase ) > 1 else answers[0]

# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


__lowerCamelCase     :					List[str]				  =						logging.get_logger(__name__)


if is_torch_available():
   import torch
   from torch import nn
   from torch.utils.data import Dataset

   from transformers import Trainer







   class __snake_case							(		lowerCamelCase_						):




      def __init__(       self     :							Tuple			,      _lowercase     :							int = 1_01 ):



        """simple docstring"""






        SCREAMING_SNAKE_CASE__     				= length




      def __len__(       self     :							str ):



        """simple docstring"""






        return self.length




      def __getitem__(       self     :							Optional[Any]			,      _lowercase     :							Union[str, Any] ):



        """simple docstring"""






        return i







   class __snake_case							:




      def __call__(       self     :							Dict			,      _lowercase     :							Dict ):



        """simple docstring"""






        return {"input_ids": torch.tensor(_lowercase ), "labels": torch.tensor(_lowercase )}







   class __snake_case							(		nn.Module						):




      def __init__(       self     :							Tuple ):



        """simple docstring"""






        super().__init__()
        # Add some (unused) params otherwise DDP will complain.
        SCREAMING_SNAKE_CASE__     				= nn.Linear(1_20			,      80 )




      def       __a		(       self     :							Any			,      _lowercase     :							Union[str, Any]			,      _lowercase     :							Dict=None ):



        """simple docstring"""






        if labels is not None:
          return torch.tensor(0.0			,      device=input_ids.device ), input_ids
        else:
          return input_ids



def       __SCREAMING_SNAKE_CASE    (   )				->   Optional[Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__     				= HfArgumentParser((TrainingArguments,)      )
  sys.argv += ["--output_dir", "./examples"]
  SCREAMING_SNAKE_CASE__     				= parser.parse_args_into_dataclasses()[0]

  logger.warning(
      f"""Process rank: {training_args.local_rank}, device: {training_args.device}, """
      f"""tpu_num_cores: {training_args.tpu_num_cores}"""					,	)

  # Essentially, what we want to verify in the distributed case is
  # that we get all samples back, in the right order.
  # (this is crucial for prediction for instance)
  for dataset_length in [10_01, 2_56, 15]:
    SCREAMING_SNAKE_CASE__     				= DummyDataset(__UpperCamelCase      )

    def compute_metrics(__UpperCamelCase       :			EvalPrediction      ) -> Dict:
      SCREAMING_SNAKE_CASE__     				= list(range(len(__UpperCamelCase      )      )      )
      SCREAMING_SNAKE_CASE__     				= p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
      return {"success": success}

    SCREAMING_SNAKE_CASE__     				= Trainer(
        model=DummyModel()					,	args=__UpperCamelCase					,	data_collator=DummyDataCollator()					,	eval_dataset=__UpperCamelCase					,	compute_metrics=__UpperCamelCase					,	)
    SCREAMING_SNAKE_CASE__     				= trainer.evaluate()
    logger.info(__UpperCamelCase      )
    if metrics["eval_success"] is not True:
      logger.error(__UpperCamelCase      )
      exit(1      )

    SCREAMING_SNAKE_CASE__     				= trainer.predict(__UpperCamelCase      )
    logger.info(p.metrics      )
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics      )
      exit(1      )

    SCREAMING_SNAKE_CASE__     				= 2

    SCREAMING_SNAKE_CASE__     				= trainer.evaluate()
    logger.info(__UpperCamelCase      )
    if metrics["eval_success"] is not True:
      logger.error(__UpperCamelCase      )
      exit(1      )

    SCREAMING_SNAKE_CASE__     				= trainer.predict(__UpperCamelCase      )
    logger.info(p.metrics      )
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics      )
      exit(1      )

    SCREAMING_SNAKE_CASE__     				= None

  logger.info("""🔥 All distributed tests successful"""      )



def       __SCREAMING_SNAKE_CASE    (   __UpperCamelCase       :			Union[str, Any]      )				->   Dict:







  """simple docstring"""





  main()


if __name__ == "__main__":
   main()
