






"""simple docstring"""



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 	_UpperCamelCase ( lowercase__		,     unittest.TestCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						ElectraTokenizer
    __lowercase    :   Tuple											=						ElectraTokenizerFast
    __lowercase    :   Tuple											=						True
    __lowercase    :   List[str]											=						True
    __lowercase    :   List[str]											=						filter_non_english


    def 		A__      (      self			):
     super().setUp()

     UpperCAmelCase__ 						=				[
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     UpperCAmelCase__ 						=				os.path.join(self.tmpdirname   ,			VOCAB_FILES_NAMES["""vocab_file"""]			)
     with open(self.vocab_file   ,			"""w"""   ,			encoding="""utf-8"""			) as vocab_writer:
      vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)


    def 		A__      (      self   ,			__lowercase			):
     UpperCAmelCase__ 						=				"UNwant\u00E9d,running"
     UpperCAmelCase__ 						=				"unwanted, running"
     return input_text, output_text


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.tokenizer_class(self.vocab_file			)

     UpperCAmelCase__ 						=				tokenizer.tokenize("""UNwant\u00E9d,running"""			)
     self.assertListEqual(__lowercase   ,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]			)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase			)   ,			[9, 6, 7, 12, 10, 11]			)


    def 		A__      (      self			):
     if not self.test_rust_tokenizer:
      return

     UpperCAmelCase__ 						=				self.get_tokenizer()
     UpperCAmelCase__ 						=				self.get_rust_tokenizer()

     UpperCAmelCase__ 						=				"UNwant\u00E9d,running"

     UpperCAmelCase__ 						=				tokenizer.tokenize(__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.tokenize(__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)

     UpperCAmelCase__ 						=				tokenizer.encode(__lowercase   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.encode(__lowercase   ,			add_special_tokens=__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)

     UpperCAmelCase__ 						=				self.get_rust_tokenizer()
     UpperCAmelCase__ 						=				tokenizer.encode(__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.encode(__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)

     # With lower casing
     UpperCAmelCase__ 						=				self.get_tokenizer(do_lower_case=__lowercase			)
     UpperCAmelCase__ 						=				self.get_rust_tokenizer(do_lower_case=__lowercase			)

     UpperCAmelCase__ 						=				"UNwant\u00E9d,running"

     UpperCAmelCase__ 						=				tokenizer.tokenize(__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.tokenize(__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)

     UpperCAmelCase__ 						=				tokenizer.encode(__lowercase   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.encode(__lowercase   ,			add_special_tokens=__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)

     UpperCAmelCase__ 						=				self.get_rust_tokenizer()
     UpperCAmelCase__ 						=				tokenizer.encode(__lowercase			)
     UpperCAmelCase__ 						=				rust_tokenizer.encode(__lowercase			)
     self.assertListEqual(__lowercase   ,			__lowercase			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""			)   ,			["""ah""", """\u535A""", """\u63A8""", """zz"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)   ,			["""hello""", """!""", """how""", """are""", """you""", """?"""]			)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)   ,			["""hello"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase   ,			strip_accents=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)   ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]			)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)   ,			["""h\u00E9llo"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase   ,			strip_accents=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)   ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)   ,			["""hello"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)   ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]			)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""			)   ,			["""hello"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """			)   ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase   ,			strip_accents=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)   ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase   ,			strip_accents=__lowercase			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """			)   ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				BasicTokenizer(do_lower_case=__lowercase   ,			never_split=["""[UNK]"""]			)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""			)   ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

     UpperCAmelCase__ 						=				{}
     for i, token in enumerate(__lowercase			):
      UpperCAmelCase__ 						=				i
     UpperCAmelCase__ 						=				WordpieceTokenizer(vocab=__lowercase   ,			unk_token="""[UNK]"""			)

     self.assertListEqual(tokenizer.tokenize(""""""			)   ,			[]			)

     self.assertListEqual(tokenizer.tokenize("""unwanted running"""			)   ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]			)

     self.assertListEqual(tokenizer.tokenize("""unwantedX running"""			)   ,			["""[UNK]""", """runn""", """##ing"""]			)


    def 		A__      (      self			):
     self.assertTrue(_is_whitespace(""" """			)			)
     self.assertTrue(_is_whitespace("""\t"""			)			)
     self.assertTrue(_is_whitespace("""\r"""			)			)
     self.assertTrue(_is_whitespace("""\n"""			)			)
     self.assertTrue(_is_whitespace("""\u00A0"""			)			)

     self.assertFalse(_is_whitespace("""A"""			)			)
     self.assertFalse(_is_whitespace("""-"""			)			)


    def 		A__      (      self			):
     self.assertTrue(_is_control("""\u0005"""			)			)

     self.assertFalse(_is_control("""A"""			)			)
     self.assertFalse(_is_control(""" """			)			)
     self.assertFalse(_is_control("""\t"""			)			)
     self.assertFalse(_is_control("""\r"""			)			)


    def 		A__      (      self			):
     self.assertTrue(_is_punctuation("""-"""			)			)
     self.assertTrue(_is_punctuation("""$"""			)			)
     self.assertTrue(_is_punctuation("""`"""			)			)
     self.assertTrue(_is_punctuation("""."""			)			)

     self.assertFalse(_is_punctuation("""A"""			)			)
     self.assertFalse(_is_punctuation(""" """			)			)


    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.get_tokenizer()
     UpperCAmelCase__ 						=				self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(__lowercase			) for t in ["""Test""", """\xad""", """test"""]]   ,			[["""[UNK]"""], [], ["""[UNK]"""]]			)

     self.assertListEqual(
         [rust_tokenizer.tokenize(__lowercase			) for t in ["""Test""", """\xad""", """test"""]]   ,			[["""[UNK]"""], [], ["""[UNK]"""]]			)


    @slow
    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""			)

     UpperCAmelCase__ 						=				tokenizer.encode("""sequence builders"""   ,			add_special_tokens=__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.encode("""multi-sequence build"""   ,			add_special_tokens=__lowercase			)

     UpperCAmelCase__ 						=				tokenizer.build_inputs_with_special_tokens(__lowercase			)
     UpperCAmelCase__ 						=				tokenizer.build_inputs_with_special_tokens(__lowercase   ,			__lowercase			)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]


    def 		A__      (      self			):
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
       UpperCAmelCase__ 						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,			**__lowercase			)

       UpperCAmelCase__ 						=				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
       UpperCAmelCase__ 						=				tokenizer_r.encode_plus(
           __lowercase   ,			return_attention_mask=__lowercase   ,			return_token_type_ids=__lowercase   ,			return_offsets_mapping=__lowercase   ,			add_special_tokens=__lowercase   ,			)

       UpperCAmelCase__ 						=				tokenizer_r.do_lower_case if hasattr(__lowercase   ,			"""do_lower_case"""			) else False
       UpperCAmelCase__ 						=				(
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "A"),
               ((1, 2), ","),
               ((3, 5), "na"),
               ((5, 6), "##ï"),
               ((6, 8), "##ve"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "Allen"),
               ((21, 23), "##NL"),
               ((23, 24), "##P"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "a"),
               ((1, 2), ","),
               ((3, 8), "naive"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "allen"),
               ((21, 23), "##nl"),
               ((23, 24), "##p"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]   ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]			)			)
       self.assertEqual([e[0] for e in expected_results]   ,			tokens["""offset_mapping"""]			)






    def 		A__      (      self			):
     UpperCAmelCase__ 						=				["的", "人", "有"]
     UpperCAmelCase__ 						=				"".join(__lowercase			)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
       UpperCAmelCase__ 						=				True
       UpperCAmelCase__ 						=				self.tokenizer_class.from_pretrained(__lowercase   ,			**__lowercase			)
       UpperCAmelCase__ 						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,			**__lowercase			)

       UpperCAmelCase__ 						=				tokenizer_p.encode(__lowercase   ,			add_special_tokens=__lowercase			)
       UpperCAmelCase__ 						=				tokenizer_r.encode(__lowercase   ,			add_special_tokens=__lowercase			)

       UpperCAmelCase__ 						=				tokenizer_r.convert_ids_to_tokens(__lowercase			)
       UpperCAmelCase__ 						=				tokenizer_p.convert_ids_to_tokens(__lowercase			)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(__lowercase   ,			__lowercase			)
       self.assertListEqual(__lowercase   ,			__lowercase			)

       UpperCAmelCase__ 						=				False
       UpperCAmelCase__ 						=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,			**__lowercase			)
       UpperCAmelCase__ 						=				self.tokenizer_class.from_pretrained(__lowercase   ,			**__lowercase			)

       UpperCAmelCase__ 						=				tokenizer_r.encode(__lowercase   ,			add_special_tokens=__lowercase			)
       UpperCAmelCase__ 						=				tokenizer_p.encode(__lowercase   ,			add_special_tokens=__lowercase			)

       UpperCAmelCase__ 						=				tokenizer_r.convert_ids_to_tokens(__lowercase			)
       UpperCAmelCase__ 						=				tokenizer_p.convert_ids_to_tokens(__lowercase			)

       # it is expected that only the first Chinese character is not preceded by "##".
       UpperCAmelCase__ 						=				[
           F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowercase			)
       ]
       self.assertListEqual(__lowercase   ,			__lowercase			)
       self.assertListEqual(__lowercase   ,			__lowercase			)










import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
				import torch

				from ..models.auto.modeling_auto import (
				    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
				    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
				)

if is_tf_available() and is_tensorflow_probability_available():
				import tensorflow as tf
				import tensorflow_probability as tfp

				from ..models.auto.modeling_tf_auto import (
				    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
				    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
				)



class  _a  (					lowercase__				):






							"""simple docstring"""

							def __call__(   self					:  List[Any]				,						a					:  str=None				,						a					:  Dict=None				,						**a					:  Tuple     )						->Dict:
													# Returns tqa_pipeline_inputs of shape:
													# [
													#   {"table": pd.DataFrame, "query": List[str]},
													#   ...,
													#   {"table": pd.DataFrame, "query" : List[str]}
													# ]
													requires_backends(self				,						"pandas"     )
													import pandas as pd

													if table is None:
																			raise ValueError("Keyword argument `table` cannot be None."     )
													elif query is None:
																			if isinstance(a				,						a     ) and table.get("query"     ) is not None and table.get("table"     ) is not None:
																									SCREAMING_SNAKE_CASE__  :					int =       [table]
																			elif isinstance(a				,						a     ) and len(a     ) > 0:
																									if not all(isinstance(a				,						a     ) for d in table     ):
																															raise ValueError(
																															    f"""Keyword argument `table` should be a list of dict, but is {(type(a     ) for d in table)}"""     )

																									if table[0].get("query"     ) is not None and table[0].get("table"     ) is not None:
																															SCREAMING_SNAKE_CASE__  :					List[str] =       table
																									else:
																															raise ValueError(
																															    "If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`"
																															    f""" and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys."""     )
																			elif Dataset is not None and isinstance(a				,						a     ) or isinstance(a				,						types.GeneratorType     ):
																									return table
																			else:
																									raise ValueError(
																									    "Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but "
																									    f"""is {type(a     )})"""     )
													else:
																			SCREAMING_SNAKE_CASE__  :					Optional[Any] =       [{"table": table, "query": query}]

													for tqa_pipeline_input in tqa_pipeline_inputs:
																			if not isinstance(tqa_pipeline_input["table"]				,						pd.DataFrame     ):
																									if tqa_pipeline_input["table"] is None:
																															raise ValueError("Table cannot be None."     )

																									SCREAMING_SNAKE_CASE__  :					Dict =       pd.DataFrame(tqa_pipeline_input["table"]     )

													return tqa_pipeline_inputs







@add_end_docstrings(lowercase__				)
class  _a  (					lowercase__				):






							"""simple docstring"""

							snake_case_							=    "table,query"

							def __init__(   self					:  List[str]				,						a					:  Optional[int]=TableQuestionAnsweringArgumentHandler()				,						*a					:  List[str]				,						**a					:  Dict     )						->Any:
													super().__init__(*a				,						**a     )
													SCREAMING_SNAKE_CASE__  :					Tuple =       args_parser

													self.check_model_type(
													    dict(
													        TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
													        + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()     )
													    if self.framework == "tf"
													    else dict(
													        MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()     )     )

													SCREAMING_SNAKE_CASE__  :					str =       bool(getattr(self.model.config				,						"aggregation_labels"				,						a     )     ) and bool(
													    getattr(self.model.config				,						"num_aggregation_labels"				,						a     )     )
													SCREAMING_SNAKE_CASE__  :					int =       "tapas" if hasattr(self.model.config				,						"aggregation_labels"     ) else None

							def   A_	(   self					:  List[str]				,						**a					:  Any     )						->Optional[Any]:
													return self.model(**a     )

							def   A_	(   self					:  Any				,						**a					:  Any     )						->Dict:
													if self.framework == "pt":
																			SCREAMING_SNAKE_CASE__  :					List[Any] =       []
																			SCREAMING_SNAKE_CASE__  :					int =       []
																			SCREAMING_SNAKE_CASE__  :					List[Any] =       None
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       inputs["input_ids"].shape[0]

																			SCREAMING_SNAKE_CASE__  :					int =       inputs["input_ids"].to(self.device     )
																			SCREAMING_SNAKE_CASE__  :					int =       inputs["attention_mask"].to(self.device     )
																			SCREAMING_SNAKE_CASE__  :					Optional[Any] =       inputs["token_type_ids"].to(self.device     )
																			SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       None

																			for index in range(a     ):
																									# If sequences have already been processed, the token type IDs will be created according to the previous
																									# answer.
																									if prev_answers is not None:
																															SCREAMING_SNAKE_CASE__  :					Tuple =       token_type_ids_example[:, 3]  # shape (seq_len,)
																															SCREAMING_SNAKE_CASE__  :					Optional[Any] =       np.zeros_like(prev_labels_example.cpu().numpy()     )  # shape (seq_len,)

																															SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       token_type_ids[index]  # shape (seq_len, 7)
																															for i in range(model_labels.shape[0]     ):
																																					SCREAMING_SNAKE_CASE__  :					int =       token_type_ids_example[:, 0].tolist()[i]
																																					SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       token_type_ids_example[:, 1].tolist()[i] - 1
																																					SCREAMING_SNAKE_CASE__  :					Dict =       token_type_ids_example[:, 2].tolist()[i] - 1

																																					if row_id >= 0 and col_id >= 0 and segment_id == 1:
																																											SCREAMING_SNAKE_CASE__  :					Dict =       int(prev_answers[(col_id, row_id)]     )

																															SCREAMING_SNAKE_CASE__  :					Dict =       torch.from_numpy(a     ).type(torch.long     ).to(self.device     )

																									SCREAMING_SNAKE_CASE__  :					Optional[int] =       input_ids[index]
																									SCREAMING_SNAKE_CASE__  :					Any =       attention_mask[index]  # shape (seq_len,)
																									SCREAMING_SNAKE_CASE__  :					List[str] =       token_type_ids[index]  # shape (seq_len, 7)
																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       self.model(
																									    input_ids=input_ids_example.unsqueeze(0     )				,						attention_mask=attention_mask_example.unsqueeze(0     )				,						token_type_ids=token_type_ids_example.unsqueeze(0     )				,						)
																									SCREAMING_SNAKE_CASE__  :					Any =       outputs.logits

																									if self.aggregate:
																															all_aggregations.append(outputs.logits_aggregation     )

																									all_logits.append(a     )

																									SCREAMING_SNAKE_CASE__  :					Any =       torch.distributions.Bernoulli(logits=a     )
																									SCREAMING_SNAKE_CASE__  :					Tuple =       dist_per_token.probs * attention_mask_example.type(torch.floataa     ).to(
																									    dist_per_token.probs.device     )

																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       collections.defaultdict(a     )
																									for i, p in enumerate(probabilities.squeeze().tolist()     ):
																															SCREAMING_SNAKE_CASE__  :					List[Any] =       token_type_ids_example[:, 0].tolist()[i]
																															SCREAMING_SNAKE_CASE__  :					Dict =       token_type_ids_example[:, 1].tolist()[i] - 1
																															SCREAMING_SNAKE_CASE__  :					Optional[Any] =       token_type_ids_example[:, 2].tolist()[i] - 1
																															if col >= 0 and row >= 0 and segment_id == 1:
																																					coords_to_probs[(col, row)].append(a     )

																									SCREAMING_SNAKE_CASE__  :					Tuple =       {key: np.array(coords_to_probs[key]     ).mean() > 0.5 for key in coords_to_probs}

																			SCREAMING_SNAKE_CASE__  :					List[str] =       torch.cat(tuple(a     )				,						0     )

																			return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(a     )				,						0     ))
													else:
																			SCREAMING_SNAKE_CASE__  :					Optional[Any] =       []
																			SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       []
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       None
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       inputs["input_ids"].shape[0]

																			SCREAMING_SNAKE_CASE__  :					Tuple =       inputs["input_ids"]
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       inputs["attention_mask"]
																			SCREAMING_SNAKE_CASE__  :					List[Any] =       inputs["token_type_ids"].numpy()
																			SCREAMING_SNAKE_CASE__  :					Tuple =       None

																			for index in range(a     ):
																									# If sequences have already been processed, the token type IDs will be created according to the previous
																									# answer.
																									if prev_answers is not None:
																															SCREAMING_SNAKE_CASE__  :					Dict =       token_type_ids_example[:, 3]  # shape (seq_len,)
																															SCREAMING_SNAKE_CASE__  :					List[Any] =       np.zeros_like(a				,						dtype=np.intaa     )  # shape (seq_len,)

																															SCREAMING_SNAKE_CASE__  :					str =       token_type_ids[index]  # shape (seq_len, 7)
																															for i in range(model_labels.shape[0]     ):
																																					SCREAMING_SNAKE_CASE__  :					str =       token_type_ids_example[:, 0].tolist()[i]
																																					SCREAMING_SNAKE_CASE__  :					Any =       token_type_ids_example[:, 1].tolist()[i] - 1
																																					SCREAMING_SNAKE_CASE__  :					Dict =       token_type_ids_example[:, 2].tolist()[i] - 1

																																					if row_id >= 0 and col_id >= 0 and segment_id == 1:
																																											SCREAMING_SNAKE_CASE__  :					int =       int(prev_answers[(col_id, row_id)]     )

																															SCREAMING_SNAKE_CASE__  :					Any =       model_labels

																									SCREAMING_SNAKE_CASE__  :					Optional[Any] =       input_ids[index]
																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       attention_mask[index]  # shape (seq_len,)
																									SCREAMING_SNAKE_CASE__  :					Any =       token_type_ids[index]  # shape (seq_len, 7)
																									SCREAMING_SNAKE_CASE__  :					Any =       self.model(
																									    input_ids=np.expand_dims(a				,						axis=0     )				,						attention_mask=np.expand_dims(a				,						axis=0     )				,						token_type_ids=np.expand_dims(a				,						axis=0     )				,						)
																									SCREAMING_SNAKE_CASE__  :					str =       outputs.logits

																									if self.aggregate:
																															all_aggregations.append(outputs.logits_aggregation     )

																									all_logits.append(a     )

																									SCREAMING_SNAKE_CASE__  :					int =       tfp.distributions.Bernoulli(logits=a     )
																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       dist_per_token.probs_parameter() * tf.cast(a				,						tf.floataa     )

																									SCREAMING_SNAKE_CASE__  :					Tuple =       collections.defaultdict(a     )
																									SCREAMING_SNAKE_CASE__  :					Optional[Any] =       token_type_ids_example
																									for i, p in enumerate(tf.squeeze(a     ).numpy().tolist()     ):
																															SCREAMING_SNAKE_CASE__  :					str =       token_type_ids_example[:, 0].tolist()[i]
																															SCREAMING_SNAKE_CASE__  :					List[Any] =       token_type_ids_example[:, 1].tolist()[i] - 1
																															SCREAMING_SNAKE_CASE__  :					Optional[Any] =       token_type_ids_example[:, 2].tolist()[i] - 1
																															if col >= 0 and row >= 0 and segment_id == 1:
																																					coords_to_probs[(col, row)].append(a     )

																									SCREAMING_SNAKE_CASE__  :					int =       {key: np.array(coords_to_probs[key]     ).mean() > 0.5 for key in coords_to_probs}

																			SCREAMING_SNAKE_CASE__  :					List[Any] =       tf.concat(tuple(a     )				,						0     )

																			return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(a     )				,						0     ))

							def __call__(   self					:  Optional[Any]				,						*a					:  Any				,						**a					:  Optional[int]     )						->Dict:
													SCREAMING_SNAKE_CASE__  :					Tuple =       self._args_parser(*a				,						**a     )

													SCREAMING_SNAKE_CASE__  :					Dict =       super().__call__(a				,						**a     )
													if len(a     ) == 1:
																			return results[0]
													return results

							def   A_	(   self					:  Any				,						a					:  Dict=None				,						a					:  Dict=None				,						a					:  Union[str, Any]=None				,						**a					:  int     )						->Any:
													SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       {}
													if padding is not None:
																			SCREAMING_SNAKE_CASE__  :					Optional[int] =       padding
													if truncation is not None:
																			SCREAMING_SNAKE_CASE__  :					int =       truncation

													SCREAMING_SNAKE_CASE__  :					Optional[int] =       {}
													if sequential is not None:
																			SCREAMING_SNAKE_CASE__  :					int =       sequential
													return preprocess_params, forward_params, {}

							def   A_	(   self					:  Any				,						a					:  Any				,						a					:  Union[str, Any]=None				,						a					:  Union[str, Any]=True				,						a					:  Dict=None     )						->int:
													if truncation is None:
																			if self.type == "tapas":
																									SCREAMING_SNAKE_CASE__  :					Any =       "drop_rows_to_fit"
																			else:
																									SCREAMING_SNAKE_CASE__  :					List[str] =       "do_not_truncate"

													SCREAMING_SNAKE_CASE__,				SCREAMING_SNAKE_CASE__  :					List[Any] =       pipeline_input["table"], pipeline_input["query"]
													if table.empty:
																			raise ValueError("table is empty"     )
													if query is None or query == "":
																			raise ValueError("query is empty"     )
													SCREAMING_SNAKE_CASE__  :					Tuple =       self.tokenizer(a				,						a				,						return_tensors=self.framework				,						truncation=a				,						padding=a     )
													SCREAMING_SNAKE_CASE__  :					str =       table
													return inputs

							def   A_	(   self					:  Any				,						a					:  Tuple				,						a					:  Union[str, Any]=False     )						->Union[str, Any]:
													SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       model_inputs.pop("table"     )

													if self.type == "tapas":
																			if sequential:
																									SCREAMING_SNAKE_CASE__  :					List[str] =       self.sequential_inference(**a     )
																			else:
																									SCREAMING_SNAKE_CASE__  :					List[Any] =       self.batch_inference(**a     )
													else:
																			SCREAMING_SNAKE_CASE__  :					Optional[Any] =       self.model.generate(**a     )
													SCREAMING_SNAKE_CASE__  :					str =       {"model_inputs": model_inputs, "table": table, "outputs": outputs}
													return model_outputs






							def   A_	(   self					:  str				,						a					:  Any     )						->Tuple:
													SCREAMING_SNAKE_CASE__  :					Dict =       model_outputs["model_inputs"]
													SCREAMING_SNAKE_CASE__  :					Optional[Any] =       model_outputs["table"]
													SCREAMING_SNAKE_CASE__  :					Dict =       model_outputs["outputs"]
													if self.type == "tapas":
																			if self.aggregate:
																									SCREAMING_SNAKE_CASE__,				SCREAMING_SNAKE_CASE__  :					List[str] =       outputs[:2]
																									SCREAMING_SNAKE_CASE__  :					Optional[int] =       self.tokenizer.convert_logits_to_predictions(a				,						a				,						a     )
																									SCREAMING_SNAKE_CASE__,				SCREAMING_SNAKE_CASE__  :					Optional[Any] =       predictions
																									SCREAMING_SNAKE_CASE__  :					Tuple =       {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(a     )}

																									SCREAMING_SNAKE_CASE__  :					Tuple =       self.model.config.no_aggregation_label_index
																									SCREAMING_SNAKE_CASE__  :					List[Any] =       {
																									    i: aggregators[i] + " > " for i, pred in enumerate(a     ) if pred != no_agg_label_index
																									}
																			else:
																									SCREAMING_SNAKE_CASE__  :					Optional[Any] =       outputs[0]
																									SCREAMING_SNAKE_CASE__  :					Optional[int] =       self.tokenizer.convert_logits_to_predictions(a				,						a     )
																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       predictions[0]
																									SCREAMING_SNAKE_CASE__  :					Dict =       {}
																									SCREAMING_SNAKE_CASE__  :					Dict =       {}
																			SCREAMING_SNAKE_CASE__  :					List[str] =       []
																			for index, coordinates in enumerate(a     ):
																									SCREAMING_SNAKE_CASE__  :					int =       [table.iat[coordinate] for coordinate in coordinates]
																									SCREAMING_SNAKE_CASE__  :					int =       aggregators.get(a				,						""     )
																									SCREAMING_SNAKE_CASE__  :					Union[str, Any] =       aggregators_prefix.get(a				,						""     )
																									SCREAMING_SNAKE_CASE__  :					Any =       {
																									    "answer": aggregator_prefix + ", ".join(a     ),
																									    "coordinates": coordinates,
																									    "cells": [table.iat[coordinate] for coordinate in coordinates],
																									}
																									if aggregator:
																															SCREAMING_SNAKE_CASE__  :					Optional[Any] =       aggregator

																									answers.append(a     )
																			if len(a     ) == 0:
																									raise PipelineException("Empty answer"     )
													else:
																			SCREAMING_SNAKE_CASE__  :					List[Any] =       [{"answer": answer} for answer in self.tokenizer.batch_decode(a				,						skip_special_tokens=a     )]

													return answers if len(a     ) > 1 else answers[0]