"""simple docstring"""







import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_a	:	Union[str, Any]=					logging.get_logger(__name__)

_a	:	Any=					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_a	:	Optional[Any]=					{
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

_a	:	str=					{
    "openai-gpt": 512,
}
def     __UpperCAmelCase							(       UpperCAmelCase_       :							str    )				->     Tuple:

	'''simple docstring'''





	__snake_case     :				Dict 						=					text.strip()
	if not text:
		return []
	__snake_case     :				Dict 						=					text.split()
	return tokens

class 						UpperCamelCase     (       lowercase		):



				def __init__(self							: Optional[int]      ,    _A							: Dict=True      ,    _A							: str=None      ,    _A							: Tuple=True      ,    _A							: Union[str, Any]=None      ,    _A							: int=True      ,    )       ->      List[str]:
					if never_split is None:
						__snake_case     :				List[str] 						=					[]
					__snake_case     :				str 						=					do_lower_case
					__snake_case     :				Tuple 						=					set(_A)
					__snake_case     :				Optional[Any] 						=					tokenize_chinese_chars
					__snake_case     :				Tuple 						=					strip_accents
					__snake_case     :				Dict 						=					do_split_on_punc



				def    _lowercase	(self							: int      ,    _A							: List[str]      ,    _A							: Optional[int]=None)       ->      Tuple:
					__snake_case     :				List[Any] 						=					self.never_split.union(set(_A)) if never_split else self.never_split
					__snake_case     :				Any 						=					self._clean_text(_A)

					# This was added on November 1st, 2018 for the multilingual and Chinese
					# models. This is also applied to the English models now, but it doesn't
					# matter since the English models were not trained on any Chinese data
					# and generally don't have any Chinese data in them (there are Chinese
					# characters in the vocabulary because Wikipedia does have some Chinese
					# words in the English Wikipedia.).
					if self.tokenize_chinese_chars:
						__snake_case     :				str 						=					self._tokenize_chinese_chars(_A)
					# prevents treating the same character with different unicode codepoints as different characters
					__snake_case     :				str 						=					unicodedata.normalize('NFC'      ,    _A)
					__snake_case     :				Tuple 						=					whitespace_tokenize(_A)
					__snake_case     :				Any 						=					[]
					for token in orig_tokens:
						if token not in never_split:
							if self.do_lower_case:
								__snake_case     :				List[Any] 						=					token.lower()
								if self.strip_accents is not False:
									__snake_case     :				Union[str, Any] 						=					self._run_strip_accents(_A)
							elif self.strip_accents:
								__snake_case     :				str 						=					self._run_strip_accents(_A)
						split_tokens.extend(self._run_split_on_punc(_A      ,    _A))

					__snake_case     :				Union[str, Any] 						=					whitespace_tokenize(' '.join(_A))
					return output_tokens



				def    _lowercase	(self							: Union[str, Any]      ,    _A							: int)       ->      Optional[Any]:
					__snake_case     :				Union[str, Any] 						=					unicodedata.normalize('NFD'      ,    _A)
					__snake_case     :				Optional[int] 						=					[]
					for char in text:
						__snake_case     :				Optional[Any] 						=					unicodedata.category(_A)
						if cat == "Mn":
							continue
						output.append(_A)
					return "".join(_A)



				def    _lowercase	(self							: Any      ,    _A							: List[Any]      ,    _A							: Optional[Any]=None)       ->      Optional[int]:
					if not self.do_split_on_punc or (never_split is not None and text in never_split):
						return [text]
					__snake_case     :				List[Any] 						=					list(_A)
					__snake_case     :				List[Any] 						=					0
					__snake_case     :				Any 						=					True
					__snake_case     :				Any 						=					[]
					while i < len(_A):
						__snake_case     :				List[str] 						=					chars[i]
						if _is_punctuation(_A):
							output.append([char])
							__snake_case     :				Any 						=					True
						else:
							if start_new_word:
								output.append([])
							__snake_case     :				Any 						=					False
							output[-1].append(_A)
						i += 1

					return ["".join(_A) for x in output]



				def    _lowercase	(self							: Any      ,    _A							: List[str])       ->      List[Any]:
					__snake_case     :				Optional[Any] 						=					[]
					for char in text:
						__snake_case     :				List[Any] 						=					ord(_A)
						if self._is_chinese_char(_A):
							output.append(' ')
							output.append(_A)
							output.append(' ')
						else:
							output.append(_A)
					return "".join(_A)



				def    _lowercase	(self							: Tuple      ,    _A							: Dict)       ->      Optional[Any]:
					# This defines a "chinese character" as anything in the CJK Unicode block:
					#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
					#
					# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
					# despite its name. The modern Korean Hangul alphabet is a different block,
					# as is Japanese Hiragana and Katakana. Those alphabets are used to write
					# space-separated words, so they are not treated specially and handled
					# like the all of the other languages.
					if (
					    (cp >= 0x4E00 and cp <= 0x9FFF)
					    or (cp >= 0x3400 and cp <= 0x4DBF)  #
					    or (cp >= 0x20000 and cp <= 0x2A6DF)  #
					    or (cp >= 0x2A700 and cp <= 0x2B73F)  #
					    or (cp >= 0x2B740 and cp <= 0x2B81F)  #
					    or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
					    or (cp >= 0xF900 and cp <= 0xFAFF)
					    or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
					):  #
						return True

					return False






				def    _lowercase	(self							: Optional[int]      ,    _A							: Any)       ->      Optional[Any]:
					__snake_case     :				int 						=					[]
					for char in text:
						__snake_case     :				Tuple 						=					ord(_A)
						if cp == 0 or cp == 0xFFFD or _is_control(_A):
							continue
						if _is_whitespace(_A):
							output.append(' ')
						else:
							output.append(_A)
					return "".join(_A)







def     __UpperCAmelCase							(       UpperCAmelCase_       :							Tuple    )				->     Tuple:

	'''simple docstring'''





	__snake_case     :				Optional[int] 						=					set()
	__snake_case     :				Optional[Any] 						=					word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)    )
		__snake_case     :				Any 						=					char
	return pairs
def     __UpperCAmelCase							(       UpperCAmelCase_       :							Optional[Any]    )				->     int:

	'''simple docstring'''





	__snake_case     :				List[Any] 						=					text.replace('—'     ,				'-'    )
	__snake_case     :				List[Any] 						=					text.replace('–'     ,				'-'    )
	__snake_case     :				Optional[int] 						=					text.replace('―'     ,				'-'    )
	__snake_case     :				Any 						=					text.replace('…'     ,				'...'    )
	__snake_case     :				str 						=					text.replace('´'     ,				'\''    )
	__snake_case     :				Tuple 						=					re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'     ,				r' \1 '     ,				UpperCAmelCase_    )
	__snake_case     :				Tuple 						=					re.sub(r'\s*\n\s*'     ,				' \n '     ,				UpperCAmelCase_    )
	__snake_case     :				List[Any] 						=					re.sub(r'[^\S\n]+'     ,				' '     ,				UpperCAmelCase_    )
	return text.strip()

class 						UpperCamelCase     (       lowercase		):
				UpperCAmelCase							: Tuple       =						VOCAB_FILES_NAMES
				UpperCAmelCase							: Dict       =						PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase							: str       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase							: Optional[int]       =						["""input_ids""", """attention_mask"""]



				def __init__(self							: Optional[int]      ,    _A							: int      ,    _A							: Dict      ,    _A							: List[Any]="<unk>"      ,    **_A							: Any)       ->      Tuple:
					super().__init__(unk_token=_A      ,    **_A)

					try:
						import ftfy
						from spacy.lang.en import English

						__snake_case     :				str 						=					English()
						__snake_case     :				Optional[int] 						=					_nlp.tokenizer
						__snake_case     :				Tuple 						=					ftfy.fix_text
					except ImportError:
						logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
						__snake_case     :				Any 						=					BasicTokenizer(do_lower_case=_A)
						__snake_case     :				Optional[Any] 						=					None

					with open(_A      ,    encoding='utf-8') as vocab_handle:
						__snake_case     :				str 						=					json.load(_A)
					__snake_case     :				Tuple 						=					{v: k for k, v in self.encoder.items()}
					with open(_A      ,    encoding='utf-8') as merges_handle:
						__snake_case     :				List[Any] 						=					merges_handle.read().split('\n')[1:-1]
					__snake_case     :				str 						=					[tuple(merge.split()) for merge in merges]
					__snake_case     :				Optional[int] 						=					dict(zip(_A      ,    range(len(_A))))
					__snake_case     :				Any 						=					{}



				@property
				def    _lowercase	(self							: Union[str, Any])       ->      Optional[Any]:
					return True



				@property
				def    _lowercase	(self							: int)       ->      Dict:
					return len(self.encoder)



				def    _lowercase	(self							: List[Any])       ->      str:
					return dict(self.encoder      ,    **self.added_tokens_encoder)



				def    _lowercase	(self							: Optional[int]      ,    _A							: int)       ->      Dict:
					__snake_case     :				List[Any] 						=					tuple(token[:-1]) + (token[-1] + '</w>',)
					if token in self.cache:
						return self.cache[token]
					__snake_case     :				Optional[int] 						=					get_pairs(_A)

					if not pairs:
						return token + "</w>"

					while True:
						__snake_case     :				Dict 						=					min(_A      ,    key=lambda _A: self.bpe_ranks.get(_A      ,    float('inf')))
						if bigram not in self.bpe_ranks:
							break
						__snake_case						,							__snake_case     :				Tuple 						=					bigram
						__snake_case     :				Dict 						=					[]
						__snake_case     :				Dict 						=					0
						while i < len(_A):
							try:
								__snake_case     :				Optional[Any] 						=					word.index(_A      ,    _A)
							except ValueError:
								new_word.extend(word[i:])
								break
							else:
								new_word.extend(word[i:j])
								__snake_case     :				Dict 						=					j

							if word[i] == first and i < len(_A) - 1 and word[i + 1] == second:
								new_word.append(first + second)
								i += 2
							else:
								new_word.append(word[i])
								i += 1
						__snake_case     :				int 						=					tuple(_A)
						__snake_case     :				Dict 						=					new_word
						if len(_A) == 1:
							break
						else:
							__snake_case     :				Union[str, Any] 						=					get_pairs(_A)
					__snake_case     :				str 						=					' '.join(_A)
					if word == "\n  </w>":
						__snake_case     :				Tuple 						=					'\n</w>'
					__snake_case     :				Any 						=					word
					return word



				def    _lowercase	(self							: Dict      ,    _A							: str)       ->      Optional[int]:
					__snake_case     :				Optional[Any] 						=					[]
					if self.fix_text is None:
						# Using BERT's BasicTokenizer
						__snake_case     :				Optional[int] 						=					self.nlp.tokenize(_A)
						for token in text:
							split_tokens.extend(list(self.bpe(_A).split(' ')))
					else:
						# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
						__snake_case     :				str 						=					self.nlp(text_standardize(self.fix_text(_A)))
						for token in text:
							split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
					return split_tokens



				def    _lowercase	(self							: Dict      ,    _A							: Any)       ->      Union[str, Any]:
					return self.encoder.get(_A      ,    self.encoder.get(self.unk_token))



				def    _lowercase	(self							: List[Any]      ,    _A							: Tuple)       ->      List[Any]:
					return self.decoder.get(_A      ,    self.unk_token)



				def    _lowercase	(self							: List[Any]      ,    _A							: Optional[int])       ->      List[str]:
					__snake_case     :				List[str] 						=					''.join(_A).replace('</w>'      ,    ' ').strip()
					return out_string






				def    _lowercase	(self							: Optional[int]      ,    _A							: str      ,    _A							: Optional[str] = None)       ->      Tuple[str]:
					if not os.path.isdir(_A):
						logger.error(f"Vocabulary path ({save_directory}) should be a directory")
						return
					__snake_case     :				Any 						=					os.path.join(
					    _A      ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
					__snake_case     :				List[Any] 						=					os.path.join(
					    _A      ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

					with open(_A      ,    'w'      ,    encoding='utf-8') as f:
						f.write(json.dumps(self.encoder      ,    indent=2      ,    sort_keys=_A      ,    ensure_ascii=_A) + '\n')

					__snake_case     :				Dict 						=					0
					with open(_A      ,    'w'      ,    encoding='utf-8') as writer:
						writer.write('#version: 0.2\n')
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,    key=lambda _A: kv[1]):
							if index != token_index:
								logger.warning(
								    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
								    ' Please check that the tokenizer is not corrupted!')
								__snake_case     :				str 						=					token_index
							writer.write(' '.join(_A) + '\n')
							index += 1

					return vocab_file, merge_file






"""simple docstring"""







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


_a	:	Union[str, Any]=					"Here is a text containing a lot of information: '''{text}'''.\n\nCan you answer this question about the text: '{question}'"

class 						UpperCamelCase     (       lowercase		):
				UpperCAmelCase							: str       =						"""google/flan-t5-base"""
				UpperCAmelCase							: int       =						(
				    """This is a tool that answers questions related to a text. It takes two arguments named `text`, which is the """
				    """text where to find the answer, and `question`, which is the question, and returns the answer to the question."""
				)
				UpperCAmelCase							: Dict       =						"""text_qa"""
				UpperCAmelCase							: Dict       =						AutoTokenizer
				UpperCAmelCase							: Optional[Any]       =						AutoModelForSeqaSeqLM

				UpperCAmelCase							: List[str]       =						["""text""", """text"""]
				UpperCAmelCase							: Any       =						["""text"""]



				def    _lowercase	(self							: Tuple      ,    _A							: str      ,    _A							: str)       ->      Tuple:
					__snake_case     :				List[Any] 						=					QA_PROMPT.format(text=_A      ,    question=_A)
					return self.pre_processor(_A      ,    return_tensors='pt')



				def    _lowercase	(self							: Tuple      ,    _A							: Optional[Any])       ->      List[Any]:
					__snake_case     :				List[Any] 						=					self.model.generate(**_A)

					__snake_case						,							__snake_case     :				Any 						=					inputs['input_ids'].shape
					__snake_case     :				Any 						=					output_ids.shape[0]

					return output_ids.reshape(_A      ,    out_b // in_b      ,    *output_ids.shape[1:])[0][0]






				def    _lowercase	(self							: Optional[int]      ,    _A							: int)       ->      str:
					return self.pre_processor.decode(_A      ,    skip_special_tokens=_A      ,    clean_up_tokenization_spaces=_A)





