






"""simple docstring"""



import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


A_     =				logging.get_logger(__name__)

A_     =				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

A_     =				{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

A_     =				{
    '''openai-gpt''': 512,
}




def   _lowerCAmelCase		(				UpperCAmelCase__							:       str   )		->str:
						A__    :	Optional[Any]			=	text.strip()
						if not text:
												return []
						A__    :	str			=	text.split()
						return tokens






class 				__SCREAMING_SNAKE_CASE    (					UpperCamelCase		):


							def __init__(      self   :    str      ,     snake_case   :    Union[str, Any]=True      ,     snake_case   :    List[Any]=None      ,     snake_case   :    List[Any]=True      ,     snake_case   :    List[str]=None      ,     snake_case   :    List[str]=True      ,     ):
													'''simple docstring'''





													if never_split is None:
																			A__    :	Any			=	[]
													A__    :	Optional[Any]			=	do_lower_case
													A__    :	Any			=	set(snake_case				)
													A__    :	Union[str, Any]			=	tokenize_chinese_chars
													A__    :	str			=	strip_accents
													A__    :	Optional[int]			=	do_split_on_punc


							def     _UpperCamelCase							(      self   :    List[str]      ,     snake_case   :    List[Any]      ,     snake_case   :    Optional[Any]=None				):
													'''simple docstring'''





													A__    :	List[Any]			=	self.never_split.union(set(snake_case				)				) if never_split else self.never_split
													A__    :	List[str]			=	self._clean_text(snake_case				)

													# This was added on November 1st, 2018 for the multilingual and Chinese
													# models. This is also applied to the English models now, but it doesn't
													# matter since the English models were not trained on any Chinese data
													# and generally don't have any Chinese data in them (there are Chinese
													# characters in the vocabulary because Wikipedia does have some Chinese
													# words in the English Wikipedia.).
													if self.tokenize_chinese_chars:
																			A__    :	List[str]			=	self._tokenize_chinese_chars(snake_case				)
													# prevents treating the same character with different unicode codepoints as different characters
													A__    :	List[Any]			=	unicodedata.normalize("""NFC"""      ,     snake_case				)
													A__    :	Any			=	whitespace_tokenize(snake_case				)
													A__    :	List[str]			=	[]
													for token in orig_tokens:
																			if token not in never_split:
																									if self.do_lower_case:
																															A__    :	Dict			=	token.lower()
																															if self.strip_accents is not False:
																																					A__    :	Dict			=	self._run_strip_accents(snake_case				)
																									elif self.strip_accents:
																															A__    :	Optional[Any]			=	self._run_strip_accents(snake_case				)
																			split_tokens.extend(self._run_split_on_punc(snake_case      ,     snake_case				)				)

													A__    :	int			=	whitespace_tokenize(""" """.join(snake_case				)				)
													return output_tokens


							def     _UpperCamelCase							(      self   :    List[Any]      ,     snake_case   :    int				):
													'''simple docstring'''





													A__    :	List[str]			=	unicodedata.normalize("""NFD"""      ,     snake_case				)
													A__    :	List[str]			=	[]
													for char in text:
																			A__    :	int			=	unicodedata.category(snake_case				)
																			if cat == "Mn":
																									continue
																			output.append(snake_case				)
													return "".join(snake_case				)


							def     _UpperCamelCase							(      self   :    Union[str, Any]      ,     snake_case   :    List[Any]      ,     snake_case   :    int=None				):
													'''simple docstring'''





													if not self.do_split_on_punc or (never_split is not None and text in never_split):
																			return [text]
													A__    :	List[str]			=	list(snake_case				)
													A__    :	Union[str, Any]			=	0
													A__    :	List[str]			=	True
													A__    :	int			=	[]
													while i < len(snake_case				):
																			A__    :	List[Any]			=	chars[i]
																			if _is_punctuation(snake_case				):
																									output.append([char]				)
																									A__    :	str			=	True
																			else:
																									if start_new_word:
																															output.append([]				)
																									A__    :	str			=	False
																									output[-1].append(snake_case				)
																			i += 1

													return ["".join(snake_case				) for x in output]


							def     _UpperCamelCase							(      self   :    int      ,     snake_case   :    Tuple				):
													'''simple docstring'''





													A__    :	Any			=	[]
													for char in text:
																			A__    :	List[Any]			=	ord(snake_case				)
																			if self._is_chinese_char(snake_case				):
																									output.append(""" """				)
																									output.append(snake_case				)
																									output.append(""" """				)
																			else:
																									output.append(snake_case				)
													return "".join(snake_case				)


							def     _UpperCamelCase							(      self   :    List[str]      ,     snake_case   :    str				):
													'''simple docstring'''





													if (
													    (cp >= 0x4E00 and cp <= 0x9FFF)
													    or (cp >= 0x3400 and cp <= 0x4DBF)  #
													    or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
													    or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
													    or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
													    or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
													    or (cp >= 0xF900 and cp <= 0xFAFF)
													    or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
													):  #
																			return True

													return False


							def     _UpperCamelCase							(      self   :    int      ,     snake_case   :    Union[str, Any]				):
													'''simple docstring'''





													A__    :	List[Any]			=	[]
													for char in text:
																			A__    :	Dict			=	ord(snake_case				)
																			if cp == 0 or cp == 0xFFFD or _is_control(snake_case				):
																									continue
																			if _is_whitespace(snake_case				):
																									output.append(""" """				)
																			else:
																									output.append(snake_case				)
													return "".join(snake_case				)




def   _lowerCAmelCase		(				UpperCAmelCase__							:       List[Any]   )		->List[Any]:
						A__    :	Optional[Any]			=	set()
						A__    :	int			=	word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)   )
												A__    :	Union[str, Any]			=	char
						return pairs




def   _lowerCAmelCase		(				UpperCAmelCase__							:       List[str]   )		->Tuple:
						A__    :	List[str]			=	text.replace("""—""",    """-"""   )
						A__    :	Optional[int]			=	text.replace("""–""",    """-"""   )
						A__    :	List[str]			=	text.replace("""―""",    """-"""   )
						A__    :	int			=	text.replace("""…""",    """..."""   )
						A__    :	str			=	text.replace("""´""",    """'"""   )
						A__    :	List[Any]			=	re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""",    R""" \1 """,    UpperCAmelCase__   )
						A__    :	Tuple			=	re.sub(R"""\s*\n\s*""",    """ \n """,    UpperCAmelCase__   )
						A__    :	Tuple			=	re.sub(R"""[^\S\n]+""",    """ """,    UpperCAmelCase__   )
						return text.strip()






class 				__SCREAMING_SNAKE_CASE    (					UpperCamelCase		):
							snake_case_  =				VOCAB_FILES_NAMES
							snake_case_  =				PRETRAINED_VOCAB_FILES_MAP
							snake_case_  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case_  =				['input_ids', 'attention_mask']


							def __init__(      self   :    List[Any]      ,     snake_case   :    List[str]      ,     snake_case   :    int      ,     snake_case   :    List[str]="<unk>"      ,     **snake_case   :    int				):
													'''simple docstring'''





													super().__init__(unk_token=snake_case      ,     **snake_case				)

													try:
																			import ftfy
																			from spacy.lang.en import English

																			A__    :	Tuple			=	English()
																			A__    :	List[Any]			=	_nlp.tokenizer
																			A__    :	Optional[Any]			=	ftfy.fix_text
													except ImportError:
																			logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""				)
																			A__    :	List[Any]			=	BasicTokenizer(do_lower_case=snake_case				)
																			A__    :	List[str]			=	None

													with open(snake_case      ,     encoding="""utf-8"""				) as vocab_handle:
																			A__    :	int			=	json.load(snake_case				)
													A__    :	Union[str, Any]			=	{v: k for k, v in self.encoder.items()}
													with open(snake_case      ,     encoding="""utf-8"""				) as merges_handle:
																			A__    :	List[str]			=	merges_handle.read().split("""\n"""				)[1:-1]
													A__    :	Optional[int]			=	[tuple(merge.split()				) for merge in merges]
													A__    :	int			=	dict(zip(snake_case      ,     range(len(snake_case				)				)				)				)
													A__    :	Union[str, Any]			=	{}


							@property
							def     _UpperCamelCase							(      self   :    List[str]				):
													'''simple docstring'''





													return True


							@property
							def     _UpperCamelCase							(      self   :    str				):
													'''simple docstring'''





													return len(self.encoder				)


							def     _UpperCamelCase							(      self   :    List[Any]				):
													'''simple docstring'''





													return dict(self.encoder      ,     **self.added_tokens_encoder				)


							def     _UpperCamelCase							(      self   :    int      ,     snake_case   :    Optional[int]				):
													'''simple docstring'''





													A__    :	str			=	tuple(token[:-1]				) + (token[-1] + """</w>""",)
													if token in self.cache:
																			return self.cache[token]
													A__    :	Union[str, Any]			=	get_pairs(snake_case				)

													if not pairs:
																			return token + "</w>"

													while True:
																			A__    :	Optional[int]			=	min(snake_case      ,     key=lambda snake_case				: self.bpe_ranks.get(snake_case      ,     float("""inf"""				)				)				)
																			if bigram not in self.bpe_ranks:
																									break
																			A__	,	A__    :	int			=	bigram
																			A__    :	Optional[int]			=	[]
																			A__    :	Tuple			=	0
																			while i < len(snake_case				):
																									try:
																															A__    :	Union[str, Any]			=	word.index(snake_case      ,     snake_case				)
																									except ValueError:
																															new_word.extend(word[i:]				)
																															break
																									else:
																															new_word.extend(word[i:j]				)
																															A__    :	List[str]			=	j

																									if word[i] == first and i < len(snake_case				) - 1 and word[i + 1] == second:
																															new_word.append(first + second				)
																															i += 2
																									else:
																															new_word.append(word[i]				)
																															i += 1
																			A__    :	Dict			=	tuple(snake_case				)
																			A__    :	Dict			=	new_word
																			if len(snake_case				) == 1:
																									break
																			else:
																									A__    :	Tuple			=	get_pairs(snake_case				)
													A__    :	str			=	""" """.join(snake_case				)
													if word == "\n  </w>":
																			A__    :	Union[str, Any]			=	"""\n</w>"""
													A__    :	Tuple			=	word
													return word


							def     _UpperCamelCase							(      self   :    int      ,     snake_case   :    Optional[int]				):
													'''simple docstring'''





													A__    :	List[str]			=	[]
													if self.fix_text is None:
																			# Using BERT's BasicTokenizer
																			A__    :	Union[str, Any]			=	self.nlp.tokenize(snake_case				)
																			for token in text:
																									split_tokens.extend(list(self.bpe(snake_case				).split(""" """				)				)				)
													else:
																			# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																			A__    :	Union[str, Any]			=	self.nlp(text_standardize(self.fix_text(snake_case				)				)				)
																			for token in text:
																									split_tokens.extend(list(self.bpe(token.text.lower()				).split(""" """				)				)				)
													return split_tokens


							def     _UpperCamelCase							(      self   :    Union[str, Any]      ,     snake_case   :    Union[str, Any]				):
													'''simple docstring'''





													return self.encoder.get(snake_case      ,     self.encoder.get(self.unk_token				)				)


							def     _UpperCamelCase							(      self   :    Union[str, Any]      ,     snake_case   :    Any				):
													'''simple docstring'''





													return self.decoder.get(snake_case      ,     self.unk_token				)


							def     _UpperCamelCase							(      self   :    Dict      ,     snake_case   :    List[str]				):
													'''simple docstring'''





													A__    :	Optional[Any]			=	"""""".join(snake_case				).replace("""</w>"""      ,     """ """				).strip()
													return out_string


							def     _UpperCamelCase							(      self   :    int      ,     snake_case   :    str      ,     snake_case   :    Optional[str] = None				):
													'''simple docstring'''





													if not os.path.isdir(snake_case				):
																			logger.error(F'Vocabulary path ({save_directory}) should be a directory'				)
																			return
													A__    :	Any			=	os.path.join(
													    snake_case      ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
													A__    :	Any			=	os.path.join(
													    snake_case      ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

													with open(snake_case      ,     """w"""      ,     encoding="""utf-8"""				) as f:
																			f.write(json.dumps(self.encoder      ,     indent=2      ,     sort_keys=snake_case      ,     ensure_ascii=snake_case				) + """\n"""				)

													A__    :	str			=	0
													with open(snake_case      ,     """w"""      ,     encoding="""utf-8"""				) as writer:
																			writer.write("""#version: 0.2\n"""				)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,     key=lambda snake_case				: kv[1]				):
																									if index != token_index:
																															logger.warning(
																															    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																															    """ Please check that the tokenizer is not corrupted!"""				)
																															A__    :	Dict			=	token_index
																									writer.write(""" """.join(snake_case				) + """\n"""				)
																									index += 1

													return vocab_file, merge_file














"""simple docstring"""



import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


A_     =				'''
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
'''






class 				__SCREAMING_SNAKE_CASE    (					unittest.TestCase							,		UpperCamelCase		):


							def     _UpperCamelCase							(      self   :    int				):
													'''simple docstring'''





													A__    :	Any			=	load_tool("""summarization"""				)
													self.tool.setup()
													A__    :	List[Any]			=	load_tool("""summarization"""      ,     remote=snake_case				)


							def     _UpperCamelCase							(      self   :    Any				):
													'''simple docstring'''





													A__    :	List[Any]			=	self.tool(snake_case				)
													self.assertEqual(
													    snake_case      ,     """Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM."""      ,     )


							def     _UpperCamelCase							(      self   :    Optional[int]				):
													'''simple docstring'''





													A__    :	List[str]			=	self.remote_tool(snake_case				)
													self.assertEqual(
													    snake_case      ,     """Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM."""      ,     )


							def     _UpperCamelCase							(      self   :    Dict				):
													'''simple docstring'''





													A__    :	List[Any]			=	self.tool(text=snake_case				)
													self.assertEqual(
													    snake_case      ,     """Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM."""      ,     )


							def     _UpperCamelCase							(      self   :    Union[str, Any]				):
													'''simple docstring'''





													A__    :	Optional[int]			=	self.remote_tool(text=snake_case				)
													self.assertEqual(
													    snake_case      ,     """Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf. In March 2021, Hugging Face raised $40 million in a Series B funding round. On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model. In 2022, the workshop concluded with the announcement of BLOOM."""      ,     )






