


"""simple docstring"""


import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
		import torch

		from ..models.auto.modeling_auto import (
		    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
		    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
		)

if is_tf_available() and is_tensorflow_probability_available():
		import tensorflow as tf
		import tensorflow_probability as tfp

		from ..models.auto.modeling_tf_auto import (
		    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
		    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
		)
class snake_case   (	_UpperCamelCase):







		def __call__(       self   :							Tuple     , a__   :							Optional[Any]=None     , a__   :							str=None     , **a__   :							Optional[int]		)							->    List[Any]:





						'''simple docstring'''







						requires_backends(self     , "pandas"		)
						import pandas as pd

						if table is None:
										raise ValueError("Keyword argument `table` cannot be None."		)
						elif query is None:
										if isinstance(a__     , a__		) and table.get("query"		) is not None and table.get("table"		) is not None:
														_A       	=      [table]
										elif isinstance(a__     , a__		) and len(a__		) > 0:
														if not all(isinstance(a__     , a__		) for d in table		):
																		raise ValueError(
																		    F"""Keyword argument `table` should be a list of dict, but is {(type(a__		) for d in table)}"""		)

														if table[0].get("query"		) is not None and table[0].get("table"		) is not None:
																		_A       	=      table
														else:
																		raise ValueError(
																		    "If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"
																		    F""" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys."""		)
										elif Dataset is not None and isinstance(a__     , a__		) or isinstance(a__     , types.GeneratorType		):
														return table
										else:
														raise ValueError(
														    "Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but "
														    F"""is {type(a__		)})"""		)
						else:
										_A       	=      [{"table": table, "query": query}]

						for tqa_pipeline_input in tqa_pipeline_inputs:
										if not isinstance(tqa_pipeline_input["table"]     , pd.DataFrame		):
														if tqa_pipeline_input["table"] is None:
																		raise ValueError("Table cannot be None."		)

														_A       	=      pd.DataFrame(tqa_pipeline_input["table"]		)

						return tqa_pipeline_inputs





@add_end_docstrings(_UpperCamelCase)
class snake_case   (	_UpperCamelCase):
		__UpperCamelCase							     =      'table,query'







		def __init__(       self   :							Dict     , a__   :							List[Any]=TableQuestionAnsweringArgumentHandler()     , *a__   :							Dict     , **a__   :							List[Any]		)							->    str:





						'''simple docstring'''







						super().__init__(*a__     , **a__		)
						_A       	=      args_parser

						self.check_model_type(
						    dict(
						        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
						        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()		)
						    if self.framework == "tf"
						    else dict(
						        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()		)		)

						_A       	=      bool(getattr(self.model.config     , "aggregation_labels"     , a__		)		) and bool(
						    getattr(self.model.config     , "num_aggregation_labels"     , a__		)		)
						_A       	=      "tapas" if hasattr(self.model.config     , "aggregation_labels"		) else None







		def 			a_			(       self   :							str     , **a__   :							List[str]		)							->    str:





						'''simple docstring'''







						return self.model(**a__		)







		def 			a_			(       self   :							int     , **a__   :							Optional[int]		)							->    Optional[Any]:





						'''simple docstring'''







						if self.framework == "pt":
										_A       	=      []
										_A       	=      []
										_A       	=      None
										_A       	=      inputs["input_ids"].shape[0]

										_A       	=      inputs["input_ids"].to(self.device		)
										_A       	=      inputs["attention_mask"].to(self.device		)
										_A       	=      inputs["token_type_ids"].to(self.device		)
										_A       	=      None

										for index in range(a__		):
														# If sequences have already been processed, the token type IDs will be created according to the previous
														# answer.
														if prev_answers is not None:
																		_A       	=      token_type_ids_example[:, 3]  # shape (seq_len,)
																		_A       	=      np.zeros_like(prev_labels_example.cpu().numpy()		)  # shape (seq_len,)

																		_A       	=      token_type_ids[index]  # shape (seq_len, 7)
																		for i in range(model_labels.shape[0]		):
																						_A       	=      token_type_ids_example[:, 0].tolist()[i]
																						_A       	=      token_type_ids_example[:, 1].tolist()[i] - 1
																						_A       	=      token_type_ids_example[:, 2].tolist()[i] - 1

																						if row_id >= 0 and col_id >= 0 and segment_id == 1:
																										_A       	=      int(prev_answers[(col_id, row_id)]		)

																		_A       	=      torch.from_numpy(a__		).type(torch.long		).to(self.device		)

														_A       	=      input_ids[index]
														_A       	=      attention_mask[index]  # shape (seq_len,)
														_A       	=      token_type_ids[index]  # shape (seq_len, 7)
														_A       	=      self.model(
														    input_ids=input_ids_example.unsqueeze(0		)     , attention_mask=attention_mask_example.unsqueeze(0		)     , token_type_ids=token_type_ids_example.unsqueeze(0		)     , )
														_A       	=      outputs.logits

														if self.aggregate:
																		all_aggregations.append(outputs.logits_aggregation		)

														all_logits.append(a__		)

														_A       	=      torch.distributions.Bernoulli(logits=a__		)
														_A       	=      dist_per_token.probs * attention_mask_example.type(torch.floataa		).to(
														    dist_per_token.probs.device		)

														_A       	=      collections.defaultdict(a__		)
														for i, p in enumerate(probabilities.squeeze().tolist()		):
																		_A       	=      token_type_ids_example[:, 0].tolist()[i]
																		_A       	=      token_type_ids_example[:, 1].tolist()[i] - 1
																		_A       	=      token_type_ids_example[:, 2].tolist()[i] - 1
																		if col >= 0 and row >= 0 and segment_id == 1:
																						coords_to_probs[(col, row)].append(a__		)

														_A       	=      {key: np.array(coords_to_probs[key]		).mean() > 0.5 for key in coords_to_probs}

										_A       	=      torch.cat(tuple(a__		)     , 0		)

										return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(a__		)     , 0		))
						else:
										_A       	=      []
										_A       	=      []
										_A       	=      None
										_A       	=      inputs["input_ids"].shape[0]

										_A       	=      inputs["input_ids"]
										_A       	=      inputs["attention_mask"]
										_A       	=      inputs["token_type_ids"].numpy()
										_A       	=      None

										for index in range(a__		):
														# If sequences have already been processed, the token type IDs will be created according to the previous
														# answer.
														if prev_answers is not None:
																		_A       	=      token_type_ids_example[:, 3]  # shape (seq_len,)
																		_A       	=      np.zeros_like(a__     , dtype=np.intaa		)  # shape (seq_len,)

																		_A       	=      token_type_ids[index]  # shape (seq_len, 7)
																		for i in range(model_labels.shape[0]		):
																						_A       	=      token_type_ids_example[:, 0].tolist()[i]
																						_A       	=      token_type_ids_example[:, 1].tolist()[i] - 1
																						_A       	=      token_type_ids_example[:, 2].tolist()[i] - 1

																						if row_id >= 0 and col_id >= 0 and segment_id == 1:
																										_A       	=      int(prev_answers[(col_id, row_id)]		)

																		_A       	=      model_labels

														_A       	=      input_ids[index]
														_A       	=      attention_mask[index]  # shape (seq_len,)
														_A       	=      token_type_ids[index]  # shape (seq_len, 7)
														_A       	=      self.model(
														    input_ids=np.expand_dims(a__     , axis=0		)     , attention_mask=np.expand_dims(a__     , axis=0		)     , token_type_ids=np.expand_dims(a__     , axis=0		)     , )
														_A       	=      outputs.logits

														if self.aggregate:
																		all_aggregations.append(outputs.logits_aggregation		)

														all_logits.append(a__		)

														_A       	=      tfp.distributions.Bernoulli(logits=a__		)
														_A       	=      dist_per_token.probs_parameter() * tf.cast(a__     , tf.floataa		)

														_A       	=      collections.defaultdict(a__		)
														_A       	=      token_type_ids_example
														for i, p in enumerate(tf.squeeze(a__		).numpy().tolist()		):
																		_A       	=      token_type_ids_example[:, 0].tolist()[i]
																		_A       	=      token_type_ids_example[:, 1].tolist()[i] - 1
																		_A       	=      token_type_ids_example[:, 2].tolist()[i] - 1
																		if col >= 0 and row >= 0 and segment_id == 1:
																						coords_to_probs[(col, row)].append(a__		)

														_A       	=      {key: np.array(coords_to_probs[key]		).mean() > 0.5 for key in coords_to_probs}

										_A       	=      tf.concat(tuple(a__		)     , 0		)

										return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(a__		)     , 0		))







		def __call__(       self   :							List[Any]     , *a__   :							Optional[int]     , **a__   :							Union[str, Any]		)							->    int:





						'''simple docstring'''







						_A       	=      self._args_parser(*a__     , **a__		)

						_A       	=      super().__call__(a__     , **a__		)
						if len(a__		) == 1:
										return results[0]
						return results







		def 			a_			(       self   :							List[str]     , a__   :							Any=None     , a__   :							Dict=None     , a__   :							List[str]=None     , **a__   :							List[str]		)							->    int:





						'''simple docstring'''







						_A       	=      {}
						if padding is not None:
										_A       	=      padding
						if truncation is not None:
										_A       	=      truncation

						_A       	=      {}
						if sequential is not None:
										_A       	=      sequential
						return preprocess_params, forward_params, {}







		def 			a_			(       self   :							Union[str, Any]     , a__   :							Optional[int]     , a__   :							Any=None     , a__   :							Dict=True     , a__   :							int=None		)							->    Union[str, Any]:





						'''simple docstring'''







						if truncation is None:
										if self.type == "tapas":
														_A       	=      "drop_rows_to_fit"
										else:
														_A       	=      "do_not_truncate"

						_A	,		_A       	=      pipeline_input["table"], pipeline_input["query"]
						if table.empty:
										raise ValueError("table is empty"		)
						if query is None or query == "":
										raise ValueError("query is empty"		)
						_A       	=      self.tokenizer(a__     , a__     , return_tensors=self.framework     , truncation=a__     , padding=a__		)
						_A       	=      table
						return inputs







		def 			a_			(       self   :							Any     , a__   :							int     , a__   :							Dict=False		)							->    Tuple:





						'''simple docstring'''







						_A       	=      model_inputs.pop("table"		)

						if self.type == "tapas":
										if sequential:
														_A       	=      self.sequential_inference(**a__		)
										else:
														_A       	=      self.batch_inference(**a__		)
						else:
										_A       	=      self.model.generate(**a__		)
						_A       	=      {"model_inputs": model_inputs, "table": table, "outputs": outputs}
						return model_outputs







		def 			a_			(       self   :							str     , a__   :							List[str]		)							->    int:





						'''simple docstring'''







						_A       	=      model_outputs["model_inputs"]
						_A       	=      model_outputs["table"]
						_A       	=      model_outputs["outputs"]
						if self.type == "tapas":
										if self.aggregate:
														_A	,		_A       	=      outputs[:2]
														_A       	=      self.tokenizer.convert_logits_to_predictions(a__     , a__     , a__		)
														_A	,		_A       	=      predictions
														_A       	=      {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(a__		)}

														_A       	=      self.model.config.no_aggregation_label_index
														_A       	=      {
														    i: aggregators[i] + " > " for i, pred in enumerate(a__		) if pred != no_agg_label_index
														}
										else:
														_A       	=      outputs[0]
														_A       	=      self.tokenizer.convert_logits_to_predictions(a__     , a__		)
														_A       	=      predictions[0]
														_A       	=      {}
														_A       	=      {}
										_A       	=      []
										for index, coordinates in enumerate(a__		):
														_A       	=      [table.iat[coordinate] for coordinate in coordinates]
														_A       	=      aggregators.get(a__     , ""		)
														_A       	=      aggregators_prefix.get(a__     , ""		)
														_A       	=      {
														    "answer": aggregator_prefix + ", ".join(a__		),
														    "coordinates": coordinates,
														    "cells": [table.iat[coordinate] for coordinate in coordinates],
														}
														if aggregator:
																		_A       	=      aggregator

														answers.append(a__		)
										if len(a__		) == 0:
														raise PipelineException("Empty answer"		)
						else:
										_A       	=      [{"answer": answer} for answer in self.tokenizer.batch_decode(a__     , skip_special_tokens=a__		)]

						return answers if len(a__		) > 1 else answers[0]



"""simple docstring"""


import os

from apache_beam.io.filesystems import FileSystems
from apache_beam.pipeline import Pipeline

from .logging import get_logger


a_											=     2 << 20  # 2mb
a_											=     get_logger(__name__)
class snake_case   (	_UpperCamelCase):







		def 			a_			(       self   :							Tuple		)							->    Optional[int]:





						'''simple docstring'''







						_A       	=      self._options.get_all_options().get("runner"		)
						return runner in [None, "DirectRunner", "PortableRunner"]






def   a__			(					__lowercase  ,      __lowercase  ,      __lowercase=False )						->						Any:
				_A       	=      FileSystems
				if fs.exists(__lowercase ):
								if force_upload:
												logger.info(f"""Remote path already exist: {remote_file_path}. Overwriting it as force_upload=True.""" )
								else:
												logger.info(f"""Remote path already exist: {remote_file_path}. Skipping it as force_upload=False.""" )
												return
				with fs.create(__lowercase ) as remote_file:
								with open(__lowercase  ,      "rb" ) as local_file:
												_A       	=      local_file.read(__lowercase )
												while chunk:
																remote_file.write(__lowercase )
																_A       	=      local_file.read(__lowercase )






def   a__			(					__lowercase  ,      __lowercase  ,      __lowercase=False )						->						Union[str, Any]:
				_A       	=      FileSystems
				if os.path.exists(__lowercase ):
								if force_download:
												logger.info(f"""Local path already exist: {remote_file_path}. Overwriting it as force_upload=True.""" )
								else:
												logger.info(f"""Local path already exist: {remote_file_path}. Skipping it as force_upload=False.""" )
												return
				with fs.open(__lowercase ) as remote_file:
								with open(__lowercase  ,      "wb" ) as local_file:
												_A       	=      remote_file.read(__lowercase )
												while chunk:
																local_file.write(__lowercase )
																_A       	=      remote_file.read(__lowercase )