






"""simple docstring"""
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


A_     :					Union[str, Any]							=     logging.get_logger(__name__)

A_     :					Any							=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

A_     :					Optional[Any]							=     {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

A_     :					str							=     {
    "openai-gpt": 5_12,
}






def 			lowerCamelCase_    (		_lowerCamelCase							):
 lowerCamelCase__	:     Dict								=		text.strip()
 if not text:
  return []
 lowerCamelCase__	:     Dict								=		text.split()
 return tokens



class 						a_ (      snake_case_					):







      '''simple docstring'''

      def __init__(self,	lowerCamelCase_=True,	lowerCamelCase_=None,	lowerCamelCase_=True,	lowerCamelCase_=None,	lowerCamelCase_=True,	):

       '''simple docstring'''



       if never_split is None:
        lowerCamelCase__	:     List[str]								=		[]
       lowerCamelCase__	:     str								=		do_lower_case
       lowerCamelCase__	:     Tuple								=		set(_A   )
       lowerCamelCase__	:     Optional[Any]								=		tokenize_chinese_chars
       lowerCamelCase__	:     Tuple								=		strip_accents
       lowerCamelCase__	:     Dict								=		do_split_on_punc




      def      a__				(self,	lowerCamelCase_,	lowerCamelCase_=None   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		self.never_split.union(set(_A   )   ) if never_split else self.never_split
       lowerCamelCase__	:     Any								=		self._clean_text(_A   )

       # This was added on November 1st, 2018 for the multilingual and Chinese
       # models. This is also applied to the English models now, but it doesn't
       # matter since the English models were not trained on any Chinese data
       # and generally don't have any Chinese data in them (there are Chinese
       # characters in the vocabulary because Wikipedia does have some Chinese
       # words in the English Wikipedia.).
       if self.tokenize_chinese_chars:
        lowerCamelCase__	:     str								=		self._tokenize_chinese_chars(_A   )
       # prevents treating the same character with different unicode codepoints as different characters
       lowerCamelCase__	:     str								=		unicodedata.normalize('NFC',	_A   )
       lowerCamelCase__	:     Tuple								=		whitespace_tokenize(_A   )
       lowerCamelCase__	:     Any								=		[]
       for token in orig_tokens:
        if token not in never_split:
         if self.do_lower_case:
          lowerCamelCase__	:     List[Any]								=		token.lower()
          if self.strip_accents is not False:
           lowerCamelCase__	:     Union[str, Any]								=		self._run_strip_accents(_A   )
         elif self.strip_accents:
          lowerCamelCase__	:     str								=		self._run_strip_accents(_A   )
        split_tokens.extend(self._run_split_on_punc(_A,	_A   )   )

       lowerCamelCase__	:     Union[str, Any]								=		whitespace_tokenize(' '.join(_A   )   )
       return output_tokens




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     Union[str, Any]								=		unicodedata.normalize('NFD',	_A   )
       lowerCamelCase__	:     Optional[int]								=		[]
       for char in text:
        lowerCamelCase__	:     Optional[Any]								=		unicodedata.category(_A   )
        if cat == "Mn":
         continue
        output.append(_A   )
       return "".join(_A   )




      def      a__				(self,	lowerCamelCase_,	lowerCamelCase_=None   ):

       '''simple docstring'''



       if not self.do_split_on_punc or (never_split is not None and text in never_split):
        return [text]
       lowerCamelCase__	:     List[Any]								=		list(_A   )
       lowerCamelCase__	:     List[Any]								=		0
       lowerCamelCase__	:     Any								=		True
       lowerCamelCase__	:     Any								=		[]
       while i < len(_A   ):
        lowerCamelCase__	:     List[str]								=		chars[i]
        if _is_punctuation(_A   ):
         output.append([char]   )
         lowerCamelCase__	:     Any								=		True
        else:
         if start_new_word:
          output.append([]   )
         lowerCamelCase__	:     Any								=		False
         output[-1].append(_A   )
        i += 1

       return ["".join(_A   ) for x in output]




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		[]
       for char in text:
        lowerCamelCase__	:     List[Any]								=		ord(_A   )
        if self._is_chinese_char(_A   ):
         output.append(' '   )
         output.append(_A   )
         output.append(' '   )
        else:
         output.append(_A   )
       return "".join(_A   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       if (
           (cp >= 0x4E00 and cp <= 0x9FFF)
           or (cp >= 0x3400 and cp <= 0x4DBF)  #
           or (cp >= 0x20000 and cp <= 0x2A6DF)  #
           or (cp >= 0x2A700 and cp <= 0x2B73F)  #
           or (cp >= 0x2B740 and cp <= 0x2B81F)  #
           or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
           or (cp >= 0xF900 and cp <= 0xFAFF)
           or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
       ):  #
        return True

       return False





      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     int								=		[]
       for char in text:
        lowerCamelCase__	:     Tuple								=		ord(_A   )
        if cp == 0 or cp == 0xFFFD or _is_control(_A   ):
         continue
        if _is_whitespace(_A   ):
         output.append(' '   )
        else:
         output.append(_A   )
       return "".join(_A   )






def 			lowerCamelCase_    (		_lowerCamelCase							):
 lowerCamelCase__	:     Optional[int]								=		set()
 lowerCamelCase__	:     Optional[Any]								=		word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)							)
  lowerCamelCase__	:     Any								=		char
 return pairs






def 			lowerCamelCase_    (		_lowerCamelCase							):
 lowerCamelCase__	:     List[Any]								=		text.replace('—'     ,  '-'							)
 lowerCamelCase__	:     List[Any]								=		text.replace('–'     ,  '-'							)
 lowerCamelCase__	:     Optional[int]								=		text.replace('―'     ,  '-'							)
 lowerCamelCase__	:     Any								=		text.replace('…'     ,  '...'							)
 lowerCamelCase__	:     str								=		text.replace('´'     ,  '\''							)
 lowerCamelCase__	:     Tuple								=		re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'     ,  r' \1 '     ,  UpperCAmelCase_							)
 lowerCamelCase__	:     Tuple								=		re.sub(r'\s*\n\s*'     ,  ' \n '     ,  UpperCAmelCase_							)
 lowerCamelCase__	:     List[Any]								=		re.sub(r'[^\S\n]+'     ,  ' '     ,  UpperCAmelCase_							)
 return text.strip()





class 						a_ (      snake_case_					):







      '''simple docstring'''

      lowerCamelCase__			:							Tuple					=      VOCAB_FILES_NAMES
      lowerCamelCase__			:							Dict					=      PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__			:							str					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__			:							Optional[int]					=      ["""input_ids""", """attention_mask"""]




      def __init__(self,	lowerCamelCase_,	lowerCamelCase_,	lowerCamelCase_="<unk>",	**lowerCamelCase_   ):

       '''simple docstring'''



       super().__init__(unk_token=_A,	**_A   )

       try:
        import ftfy
        from spacy.lang.en import English

        lowerCamelCase__	:     str								=		English()
        lowerCamelCase__	:     Optional[int]								=		_nlp.tokenizer
        lowerCamelCase__	:     Tuple								=		ftfy.fix_text
       except ImportError:
        logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'   )
        lowerCamelCase__	:     Any								=		BasicTokenizer(do_lower_case=_A   )
        lowerCamelCase__	:     Optional[Any]								=		None

       with open(_A,	encoding='utf-8'   ) as vocab_handle:
        lowerCamelCase__	:     str								=		json.load(_A   )
       lowerCamelCase__	:     Tuple								=		{v: k for k, v in self.encoder.items()}
       with open(_A,	encoding='utf-8'   ) as merges_handle:
        lowerCamelCase__	:     List[Any]								=		merges_handle.read().split('\n'   )[1:-1]
       lowerCamelCase__	:     str								=		[tuple(merge.split()   ) for merge in merges]
       lowerCamelCase__	:     Optional[int]								=		dict(zip(_A,	range(len(_A   )   )   )   )
       lowerCamelCase__	:     Any								=		{}




      @property
      def      a__				(self   ):

       '''simple docstring'''



       return True




      @property
      def      a__				(self   ):

       '''simple docstring'''



       return len(self.encoder   )




      def      a__				(self   ):

       '''simple docstring'''



       return dict(self.encoder,	**self.added_tokens_encoder   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		tuple(token[:-1]   ) + (token[-1] + '</w>',)
       if token in self.cache:
        return self.cache[token]
       lowerCamelCase__	:     Optional[int]								=		get_pairs(_A   )

       if not pairs:
        return token + "</w>"

       while True:
        lowerCamelCase__	:     Dict								=		min(_A,	key=lambda lowerCamelCase_   : self.bpe_ranks.get(_A,	float('inf'   )   )   )
        if bigram not in self.bpe_ranks:
         break
        lowerCamelCase__	:     Tuple								=		bigram
        lowerCamelCase__	:     Dict								=		[]
        lowerCamelCase__	:     Dict								=		0
        while i < len(_A   ):
         try:
          lowerCamelCase__	:     Optional[Any]								=		word.index(_A,	_A   )
         except ValueError:
          new_word.extend(word[i:]   )
          break
         else:
          new_word.extend(word[i:j]   )
          lowerCamelCase__	:     Dict								=		j

         if word[i] == first and i < len(_A   ) - 1 and word[i + 1] == second:
          new_word.append(first + second   )
          i += 2
         else:
          new_word.append(word[i]   )
          i += 1
        lowerCamelCase__	:     int								=		tuple(_A   )
        lowerCamelCase__	:     Dict								=		new_word
        if len(_A   ) == 1:
         break
        else:
         lowerCamelCase__	:     Union[str, Any]								=		get_pairs(_A   )
       lowerCamelCase__	:     str								=		' '.join(_A   )
       if word == "\n  </w>":
        lowerCamelCase__	:     Tuple								=		'\n</w>'
       lowerCamelCase__	:     Any								=		word
       return word




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		[]
       if self.fix_text is None:
        # Using BERT's BasicTokenizer
        lowerCamelCase__	:     Optional[int]								=		self.nlp.tokenize(_A   )
        for token in text:
         split_tokens.extend(list(self.bpe(_A   ).split(' '   )   )   )
       else:
        # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
        lowerCamelCase__	:     str								=		self.nlp(text_standardize(self.fix_text(_A   )   )   )
        for token in text:
         split_tokens.extend(list(self.bpe(token.text.lower()   ).split(' '   )   )   )
       return split_tokens




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       return self.encoder.get(_A,	self.encoder.get(self.unk_token   )   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       return self.decoder.get(_A,	self.unk_token   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[str]								=		''.join(_A   ).replace('</w>',	' '   ).strip()
       return out_string





      def      a__				(self,	lowerCamelCase_,	lowerCamelCase_ = None   ):

       '''simple docstring'''



       if not os.path.isdir(_A   ):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
        return
       lowerCamelCase__	:     Any								=		os.path.join(
           _A,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )
       lowerCamelCase__	:     List[Any]								=		os.path.join(
           _A,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

       with open(_A,	'w',	encoding='utf-8'   ) as f:
        f.write(json.dumps(self.encoder,	indent=2,	sort_keys=_A,	ensure_ascii=_A   ) + '\n'   )

       lowerCamelCase__	:     Dict								=		0
       with open(_A,	'w',	encoding='utf-8'   ) as writer:
        writer.write('#version: 0.2\n'   )
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),	key=lambda lowerCamelCase_   : kv[1]   ):
         if index != token_index:
          logger.warning(
              f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              ' Please check that the tokenizer is not corrupted!'   )
          lowerCamelCase__	:     str								=		token_index
         writer.write(' '.join(_A   ) + '\n'   )
         index += 1

       return vocab_file, merge_file


"""simple docstring"""







import dataclasses
import enum
import functools
import math
import re

# The following script is adapted from the script of TaPas.
# Original: https://github.com/google-research/tapas/master/wikisql_utils.py
from typing import Any, List, Text


_a	:	List[Any]=					"none"
_a	:	List[str]=					"none"
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Optional[int]     ,				UpperCAmelCase_       :							Tuple    )				->     Dict:

	'''simple docstring'''





	__snake_case     :				int 						=					value.split(UpperCAmelCase_    )
	return len(UpperCAmelCase_    ) > 1 and any((len(UpperCAmelCase_    ) == 3 for x in split)    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Optional[Any]    )				->     Tuple:

	'''simple docstring'''





	if isinstance(UpperCAmelCase_     ,				UpperCAmelCase_    ):
		return value
	if isinstance(UpperCAmelCase_     ,				UpperCAmelCase_    ):
		return float(UpperCAmelCase_    )
	if not isinstance(UpperCAmelCase_     ,				UpperCAmelCase_    ):
		raise ValueError('Argument value is not a string. Can\'t parse it as float'    )
	__snake_case     :				Any 						=					value

	try:
		# Example: 1,000.7
		if "." in sanitized and "," in sanitized:
			return float(sanitized.replace(','     ,				''    )    )
		# 1,000
		if "," in sanitized and _split_thousands(','     ,				UpperCAmelCase_    ):
			return float(sanitized.replace(','     ,				''    )    )
		# 5,5556
		if "," in sanitized and sanitized.count(','    ) == 1 and not _split_thousands(','     ,				UpperCAmelCase_    ):
			return float(sanitized.replace(','     ,				'.'    )    )
		# 0.0.0.1
		if sanitized.count('.'    ) > 1:
			return float(sanitized.replace('.'     ,				''    )    )
		# 0,0,0,1
		if sanitized.count(','    ) > 1:
			return float(sanitized.replace(','     ,				''    )    )
		return float(UpperCAmelCase_    )
	except ValueError:
		# Avoid adding the sanitized value in the error message.
		raise ValueError('Unable to convert value to float'    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							str    )				->     Any:

	'''simple docstring'''





	if answer is None:
		return None
	try:
		__snake_case     :				Dict 						=					convert_to_float(UpperCAmelCase_    )
		if isinstance(UpperCAmelCase_     ,				UpperCAmelCase_    ) and math.isnan(UpperCAmelCase_    ):
			return None
		return value
	except ValueError:
		return answer.lower()


_a	:	Union[str, Any]=					{
    "text": lambda x: x,
    "real": convert_to_float,
}

class 						UpperCamelCase     (       enum.Enum		):
				UpperCAmelCase							: int       =						0
				UpperCAmelCase							: Optional[Any]       =						1
				UpperCAmelCase							: List[Any]       =						2
				UpperCAmelCase							: Dict       =						3
				UpperCAmelCase							: Union[str, Any]       =						4
				UpperCAmelCase							: int       =						5







class 						UpperCamelCase     (       enum.Enum		):
				UpperCAmelCase							: List[str]       =						0
				UpperCAmelCase							: List[str]       =						1
				UpperCAmelCase							: int       =						2







@dataclasses.dataclass
class 						UpperCamelCase     :
				UpperCAmelCase							: Text
				UpperCAmelCase							: _Operator
				UpperCAmelCase							: Any







_a	:	Union[str, Any]=					re.compile(R"\w+|[^\w\s]+", re.UNICODE | re.MULTILINE | re.DOTALL)
def     __UpperCAmelCase							(       UpperCAmelCase_       :							List[str]    )				->     Optional[Any]:

	'''simple docstring'''





	return list(_TOKENIZER.findall(x.lower()    )    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Tuple     ,				UpperCAmelCase_       :							int     ,				UpperCAmelCase_       :							List[Any]    )				->     List[str]:

	'''simple docstring'''





	if operator == _Operator.EQUALS:
		return src == tgt
	elif operator == _Operator.GREATER:
		return src > tgt
	elif operator == _Operator.LESSER:
		return src < tgt
	raise ValueError(F"Unknown operator: {operator}"    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							List[Any]     ,				UpperCAmelCase_       :							int     ,				UpperCAmelCase_       :							Optional[Any]    )				->     Dict:

	'''simple docstring'''





	__snake_case     :				Optional[Any] 						=					table['types']
	return _TYPE_CONVERTER[types[column]](UpperCAmelCase_    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							List[Any]    )				->     str:

	'''simple docstring'''





	return isinstance(UpperCAmelCase_     ,				UpperCAmelCase_    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Dict     ,				UpperCAmelCase_       :							Union[str, Any]     ,				UpperCAmelCase_       :							int    )				->     List[Any]:

	'''simple docstring'''





	for cond in conditions:
		__snake_case     :				Any 						=					row[cond.column]

		__snake_case     :				Tuple 						=					_parse_value(UpperCAmelCase_     ,				cond.column     ,				cond.cmp_value    )

		if _is_string(UpperCAmelCase_    ) and _is_string(UpperCAmelCase_    ):
			__snake_case     :				str 						=					_normalize_for_match(UpperCAmelCase_    )
			__snake_case     :				Dict 						=					_normalize_for_match(UpperCAmelCase_    )

		if not isinstance(UpperCAmelCase_     ,				type(UpperCAmelCase_    )    ):
			raise ValueError('Type difference {} != {}'.format(type(UpperCAmelCase_    )     ,				type(UpperCAmelCase_    )    )    )

		if not _compare(cond.operator     ,				UpperCAmelCase_     ,				UpperCAmelCase_    ):
			return False
	return True
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Union[str, Any]     ,				UpperCAmelCase_       :							List[str]     ,				UpperCAmelCase_       :							Optional[int]    )				->     Dict:

	'''simple docstring'''





	if not answer_coordinates:
		if aggregation_op == _Aggregation.COUNT:
			return 0.0
		else:
			return EMPTY_ANSWER_AGG

    # Count can support non numeric answers.
	if aggregation_op == _Aggregation.COUNT:
		return float(len(UpperCAmelCase_    )    )

	# If we have just one answer, if float returns it or try a conversion.
	__snake_case     :				Any 						=					[table['rows'][i][j] for (i, j) in answer_coordinates]
	if len(UpperCAmelCase_    ) == 1:
		try:
			return convert_to_float(values[0]    )
		except ValueError as e:
			if aggregation_op != _Aggregation.NONE:
				raise e

	if aggregation_op == _Aggregation.NONE:
		return None

	# Other aggregation only support numeric values. Bail out if we have strings.
	if not all((isinstance(UpperCAmelCase_     ,				(int, float)    ) for v in values)    ):
		return None

	if aggregation_op == _Aggregation.SUM:
		return float(sum(UpperCAmelCase_    )    )
	elif aggregation_op == _Aggregation.AVERAGE:
		return sum(UpperCAmelCase_    ) / len(UpperCAmelCase_    )
	else:
		raise ValueError(F"Unknown aggregation: {aggregation_op}"    )
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Optional[Any]     ,				UpperCAmelCase_       :							int    )				->     str:

	'''simple docstring'''





	__snake_case     :				int 						=					sql_query['agg']
	if aggregation_op_index >= 3:
		__snake_case     :				Optional[Any] 						=					_Aggregation(UpperCAmelCase_    )
	else:
		__snake_case     :				Any 						=					_Aggregation.NONE

	__snake_case     :				Dict 						=					sql_query['sel']
	__snake_case     :				Dict 						=					[
	    _Condition(UpperCAmelCase_     ,				_Operator(UpperCAmelCase_    )     ,				UpperCAmelCase_    )
	    for column, operator, cmp_value in zip(
	        sql_query['conds']['column_index']     ,				sql_query['conds']['operator_index']     ,				sql_query['conds']['condition']    )
	]

	__snake_case     :				str 						=					[]
	for row in range(len(table['rows']    )    ):
		if _respect_conditions(UpperCAmelCase_     ,				table['rows'][row]     ,				UpperCAmelCase_    ):
			indices.append((row, target_column)    )

	if not indices:
		return [], aggregation_op

	if len(UpperCAmelCase_    ) == 1:
		return indices, aggregation_op

	# Parsing of MIN/MAX.
	if aggregation_op_index in (1, 2):
		__snake_case     :				Union[str, Any] 						=					{2: min, 1: max}
		__snake_case     :				Optional[Any] 						=					[(table['rows'][i][j], index) for index, (i, j) in enumerate(UpperCAmelCase_    )]
		__snake_case     :				Tuple 						=					functools.reduce(operators[sql_query['agg']]     ,				UpperCAmelCase_    )

		__snake_case     :				str 						=					[indices[reduced[1]]]
		return ret, _Aggregation.NONE

	return indices, aggregation_op
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Dict     ,				UpperCAmelCase_       :							List[Any]     ,				UpperCAmelCase_       :							Any    )				->     Optional[Any]:

	'''simple docstring'''





	if float_answer is not None:
		return [str(UpperCAmelCase_    )]
	return [str(table['real_rows'][r][c]    ) for r, c in answer_coordinates]

def     __UpperCAmelCase							(       UpperCAmelCase_       :							int     ,				UpperCAmelCase_       :							Tuple    )				->     List:

	'''simple docstring'''





	__snake_case						,							__snake_case     :				List[Any] 						=					_get_answer_coordinates(UpperCAmelCase_     ,				UpperCAmelCase_    )
	__snake_case     :				Optional[int] 						=					_get_float_answer(UpperCAmelCase_     ,				UpperCAmelCase_     ,				UpperCAmelCase_    )
	__snake_case     :				Union[str, Any] 						=					_get_answer_text(UpperCAmelCase_     ,				UpperCAmelCase_     ,				UpperCAmelCase_    )
	# keep the original data the same with TaPas
	if len(UpperCAmelCase_    ) == 0:
		__snake_case     :				List[str] 						=					[EMPTY_ANSWER]
	return answer_text





