




'''simple docstring'''

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class __magic_name__(						__lowerCAmelCase    ,		unittest.TestCase      ):
						UpperCAmelCase_		:					List[str]    =  RealmTokenizer
						UpperCAmelCase_		:					str    =  RealmTokenizerFast
						UpperCAmelCase_		:					Dict    =  True
						UpperCAmelCase_		:					str    =  True
						UpperCAmelCase_		:					Union[str, Any]    =  filter_non_english






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									super().setUp()

									snake_case__          =						[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									snake_case__          =						os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["""vocab_file"""]       )
									with open(self.vocab_file					,   """w"""					,   encoding="""utf-8"""       ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






						def       __lowerCAmelCase(   self						:						Tuple					,   __UpperCamelCase						:						int       ):



									'''simple docstring'''






									snake_case__          =						"""UNwant\u00E9d,running"""
									snake_case__          =						"""unwanted, running"""
									return input_text, output_text






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						self.tokenizer_class(self.vocab_file       )

									snake_case__          =						tokenizer.tokenize("""UNwant\u00E9d,running"""       )
									self.assertListEqual(__UpperCamelCase					,   ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]       )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase       )					,   [9, 6, 7, 1_2, 1_0, 1_1]       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									if not self.test_rust_tokenizer:
												return

									snake_case__          =						self.get_tokenizer()
									snake_case__          =						self.get_rust_tokenizer()

									snake_case__          =						"""UNwant\u00E9d,running"""

									snake_case__          =						tokenizer.tokenize(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.tokenize(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						self.get_rust_tokenizer()
									snake_case__          =						tokenizer.encode(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									# With lower casing
									snake_case__          =						self.get_tokenizer(do_lower_case=__UpperCamelCase       )
									snake_case__          =						self.get_rust_tokenizer(do_lower_case=__UpperCamelCase       )

									snake_case__          =						"""UNwant\u00E9d,running"""

									snake_case__          =						tokenizer.tokenize(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.tokenize(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase					,   add_special_tokens=__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )

									snake_case__          =						self.get_rust_tokenizer()
									snake_case__          =						tokenizer.encode(__UpperCamelCase       )
									snake_case__          =						rust_tokenizer.encode(__UpperCamelCase       )
									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       )					,   ["""ah""", """\u535A""", """\u63A8""", """zz"""]       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )					,   ["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""h\u00E9llo"""]       )






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )					,   ["""hello"""]       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )					,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						Dict       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   strip_accents=__UpperCamelCase       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )					,   ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase					,   never_split=["""[UNK]"""]       )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       )					,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									snake_case__          =						{}
									for i, token in enumerate(__UpperCamelCase       ):
												snake_case__          =						i
									snake_case__          =						WordpieceTokenizer(vocab=__UpperCamelCase					,   unk_token="""[UNK]"""       )

									self.assertListEqual(tokenizer.tokenize(""""""       )					,   []       )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""       )					,   ["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       )					,   ["""[UNK]""", """runn""", """##ing"""]       )






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									self.assertTrue(_is_whitespace(""" """       )       )
									self.assertTrue(_is_whitespace("""\t"""       )       )
									self.assertTrue(_is_whitespace("""\r"""       )       )
									self.assertTrue(_is_whitespace("""\n"""       )       )
									self.assertTrue(_is_whitespace("""\u00A0"""       )       )

									self.assertFalse(_is_whitespace("""A"""       )       )
									self.assertFalse(_is_whitespace("""-"""       )       )






						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									self.assertTrue(_is_control("""\u0005"""       )       )

									self.assertFalse(_is_control("""A"""       )       )
									self.assertFalse(_is_control(""" """       )       )
									self.assertFalse(_is_control("""\t"""       )       )
									self.assertFalse(_is_control("""\r"""       )       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									self.assertTrue(_is_punctuation("""-"""       )       )
									self.assertTrue(_is_punctuation("""$"""       )       )
									self.assertTrue(_is_punctuation("""`"""       )       )
									self.assertTrue(_is_punctuation("""."""       )       )

									self.assertFalse(_is_punctuation("""A"""       )       )
									self.assertFalse(_is_punctuation(""" """       )       )






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(__UpperCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]					,   [["""[UNK]"""], [], ["""[UNK]"""]]       )

									if self.test_rust_tokenizer:
												snake_case__          =						self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(__UpperCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]					,   [["""[UNK]"""], [], ["""[UNK]"""]]       )






						@slow
						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						self.tokenizer_class.from_pretrained("""bert-base-uncased"""       )

									snake_case__          =						tokenizer.encode("""sequence builders"""					,   add_special_tokens=__UpperCamelCase       )
									snake_case__          =						tokenizer.encode("""multi-sequence build"""					,   add_special_tokens=__UpperCamelCase       )

									snake_case__          =						tokenizer.build_inputs_with_special_tokens(__UpperCamelCase       )
									snake_case__          =						tokenizer.build_inputs_with_special_tokens(__UpperCamelCase					,   __UpperCamelCase       )

									assert encoded_sentence == [1_0_1] + text + [1_0_2]
									assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
															snake_case__          =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )

															snake_case__          =						f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
															snake_case__          =						tokenizer_r.encode_plus(
															    __UpperCamelCase					,   return_attention_mask=__UpperCamelCase					,   return_token_type_ids=__UpperCamelCase					,   return_offsets_mapping=__UpperCamelCase					,   add_special_tokens=__UpperCamelCase					,   )

															snake_case__          =						tokenizer_r.do_lower_case if hasattr(__UpperCamelCase					,   """do_lower_case"""       ) else False
															snake_case__          =						(
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """A"""),
															        ((1, 2), ""","""),
															        ((3, 5), """na"""),
															        ((5, 6), """##ï"""),
															        ((6, 8), """##ve"""),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), """Allen"""),
															        ((2_1, 2_3), """##NL"""),
															        ((2_3, 2_4), """##P"""),
															        ((2_5, 3_3), """sentence"""),
															        ((3_3, 3_4), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), """a"""),
															        ((1, 2), ""","""),
															        ((3, 8), """naive"""),
															        ((9, 1_5), tokenizer_r.mask_token),
															        ((1_6, 2_1), """allen"""),
															        ((2_1, 2_3), """##nl"""),
															        ((2_3, 2_4), """##p"""),
															        ((2_5, 3_3), """sentence"""),
															        ((3_3, 3_4), """."""),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]					,   tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
															self.assertEqual([e[0] for e in expected_results]					,   tokens["""offset_mapping"""]       )







						@slow
						def       __lowerCAmelCase(   self						:						Tuple       ):



									'''simple docstring'''






									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
															snake_case__          =						self.rust_tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )
															snake_case__          =						self.tokenizer_class.from_pretrained(__UpperCamelCase					,   **__UpperCamelCase       )
															snake_case__          =						[["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]

															snake_case__          =						tokenizer_r.batch_encode_candidates(__UpperCamelCase					,   max_length=1_0					,   return_tensors="""np"""       )
															snake_case__          =						tokenizer_p.batch_encode_candidates(__UpperCamelCase					,   max_length=1_0					,   return_tensors="""np"""       )

															snake_case__          =						(2, 2, 1_0)

															self.assertEqual(encoded_sentence_r["""input_ids"""].shape					,   __UpperCamelCase       )
															self.assertEqual(encoded_sentence_r["""attention_mask"""].shape					,   __UpperCamelCase       )
															self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape					,   __UpperCamelCase       )

															self.assertEqual(encoded_sentence_p["""input_ids"""].shape					,   __UpperCamelCase       )
															self.assertEqual(encoded_sentence_p["""attention_mask"""].shape					,   __UpperCamelCase       )
															self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape					,   __UpperCamelCase       )





'''simple docstring'''

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a__         =     logging.get_logger(__name__)

a__         =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

a__         =     {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

a__         =     {
    '''openai-gpt''': 512,
}




def    snake_case__     (  a	)			->       Tuple:





			'''simple docstring'''






			snake_case__          =						text.strip()
			if not text:
						return []
			snake_case__          =						text.split()
			return tokens





class __magic_name__(						__lowerCAmelCase      ):






						def __init__(   self						:						Any					,   __UpperCamelCase						:						str=True					,   __UpperCamelCase						:						Dict=None					,   __UpperCamelCase						:						Union[str, Any]=True					,   __UpperCamelCase						:						List[str]=None					,   __UpperCamelCase						:						Optional[int]=True					,   ):



									'''simple docstring'''






									if never_split is None:
												snake_case__          =						[]
									snake_case__          =						do_lower_case
									snake_case__          =						set(__UpperCamelCase       )
									snake_case__          =						tokenize_chinese_chars
									snake_case__          =						strip_accents
									snake_case__          =						do_split_on_punc






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						Any=None       ):



									'''simple docstring'''






									snake_case__          =						self.never_split.union(set(__UpperCamelCase       )       ) if never_split else self.never_split
									snake_case__          =						self._clean_text(__UpperCamelCase       )

									# This was added on November 1st, 2018 for the multilingual and Chinese
									# models. This is also applied to the English models now, but it doesn't
									# matter since the English models were not trained on any Chinese data
									# and generally don't have any Chinese data in them (there are Chinese
									# characters in the vocabulary because Wikipedia does have some Chinese
									# words in the English Wikipedia.).
									if self.tokenize_chinese_chars:
												snake_case__          =						self._tokenize_chinese_chars(__UpperCamelCase       )
									# prevents treating the same character with different unicode codepoints as different characters
									snake_case__          =						unicodedata.normalize("""NFC"""					,   __UpperCamelCase       )
									snake_case__          =						whitespace_tokenize(__UpperCamelCase       )
									snake_case__          =						[]
									for token in orig_tokens:
												if token not in never_split:
															if self.do_lower_case:
																		snake_case__          =						token.lower()
																		if self.strip_accents is not False:
																					snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
															elif self.strip_accents:
																		snake_case__          =						self._run_strip_accents(__UpperCamelCase       )
												split_tokens.extend(self._run_split_on_punc(__UpperCamelCase					,   __UpperCamelCase       )       )

									snake_case__          =						whitespace_tokenize(""" """.join(__UpperCamelCase       )       )
									return output_tokens






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Tuple       ):



									'''simple docstring'''






									snake_case__          =						unicodedata.normalize("""NFD"""					,   __UpperCamelCase       )
									snake_case__          =						[]
									for char in text:
												snake_case__          =						unicodedata.category(__UpperCamelCase       )
												if cat == "Mn":
															continue
												output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Any					,   __UpperCamelCase						:						int=None       ):



									'''simple docstring'''






									if not self.do_split_on_punc or (never_split is not None and text in never_split):
												return [text]
									snake_case__          =						list(__UpperCamelCase       )
									snake_case__          =						0
									snake_case__          =						True
									snake_case__          =						[]
									while i < len(__UpperCamelCase       ):
												snake_case__          =						chars[i]
												if _is_punctuation(__UpperCamelCase       ):
															output.append([char]       )
															snake_case__          =						True
												else:
															if start_new_word:
																		output.append([]       )
															snake_case__          =						False
															output[-1].append(__UpperCamelCase       )
												i += 1

									return ["".join(__UpperCamelCase       ) for x in output]






						def       __lowerCAmelCase(   self						:						Tuple					,   __UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if self._is_chinese_char(__UpperCamelCase       ):
															output.append(""" """       )
															output.append(__UpperCamelCase       )
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									if (
									    (cp >= 0X4_e00 and cp <= 0X9_fff)
									    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
									    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
									    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
									    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
									    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
									    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
									    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
									):  #
												return True

									return False







						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Any       ):



									'''simple docstring'''






									snake_case__          =						[]
									for char in text:
												snake_case__          =						ord(__UpperCamelCase       )
												if cp == 0 or cp == 0Xf_ffd or _is_control(__UpperCamelCase       ):
															continue
												if _is_whitespace(__UpperCamelCase       ):
															output.append(""" """       )
												else:
															output.append(__UpperCamelCase       )
									return "".join(__UpperCamelCase       )







def    snake_case__     (  a	)			->       int:





			'''simple docstring'''






			snake_case__          =						set()
			snake_case__          =						word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						snake_case__          =						char
			return pairs




def    snake_case__     (  a	)			->       Union[str, Any]:





			'''simple docstring'''






			snake_case__          =						text.replace("""—"""       ,      """-"""	)
			snake_case__          =						text.replace("""–"""       ,      """-"""	)
			snake_case__          =						text.replace("""―"""       ,      """-"""	)
			snake_case__          =						text.replace("""…"""       ,      """..."""	)
			snake_case__          =						text.replace("""´"""       ,      """'"""	)
			snake_case__          =						re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""       ,      R""" \1 """       ,      a	)
			snake_case__          =						re.sub(R"""\s*\n\s*"""       ,      """ \n """       ,      a	)
			snake_case__          =						re.sub(R"""[^\S\n]+"""       ,      """ """       ,      a	)
			return text.strip()





class __magic_name__(						__lowerCAmelCase      ):
						UpperCAmelCase_		:					str    =  VOCAB_FILES_NAMES
						UpperCAmelCase_		:					Dict    =  PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_		:					Optional[Any]    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_		:					List[str]    =  ["""input_ids""", """attention_mask"""]






						def __init__(   self						:						int					,   __UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Dict="<unk>"					,   **__UpperCamelCase						:						Optional[Any]       ):



									'''simple docstring'''






									super().__init__(unk_token=__UpperCamelCase					,   **__UpperCamelCase       )

									try:
												import ftfy
												from spacy.lang.en import English

												snake_case__          =						English()
												snake_case__          =						_nlp.tokenizer
												snake_case__          =						ftfy.fix_text
									except ImportError:
												logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""       )
												snake_case__          =						BasicTokenizer(do_lower_case=__UpperCamelCase       )
												snake_case__          =						None

									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as vocab_handle:
												snake_case__          =						json.load(__UpperCamelCase       )
									snake_case__          =						{v: k for k, v in self.encoder.items()}
									with open(__UpperCamelCase					,   encoding="""utf-8"""       ) as merges_handle:
												snake_case__          =						merges_handle.read().split("""\n"""       )[1:-1]
									snake_case__          =						[tuple(merge.split()       ) for merge in merges]
									snake_case__          =						dict(zip(__UpperCamelCase					,   range(len(__UpperCamelCase       )       )       )       )
									snake_case__          =						{}






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return True






						@property
						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									return len(self.encoder       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									return dict(self.encoder					,   **self.added_tokens_encoder       )






						def       __lowerCAmelCase(   self						:						Optional[Any]					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						tuple(token[:-1]       ) + (token[-1] + """</w>""",)
									if token in self.cache:
												return self.cache[token]
									snake_case__          =						get_pairs(__UpperCamelCase       )

									if not pairs:
												return token + "</w>"

									while True:
												snake_case__          =						min(__UpperCamelCase					,   key=lambda __UpperCamelCase       : self.bpe_ranks.get(__UpperCamelCase					,   float("""inf"""       )       )       )
												if bigram not in self.bpe_ranks:
															break
												snake_case__    ,			snake_case__          =						bigram
												snake_case__          =						[]
												snake_case__          =						0
												while i < len(__UpperCamelCase       ):
															try:
																		snake_case__          =						word.index(__UpperCamelCase					,   __UpperCamelCase       )
															except ValueError:
																		new_word.extend(word[i:]       )
																		break
															else:
																		new_word.extend(word[i:j]       )
																		snake_case__          =						j

															if word[i] == first and i < len(__UpperCamelCase       ) - 1 and word[i + 1] == second:
																		new_word.append(first + second       )
																		i += 2
															else:
																		new_word.append(word[i]       )
																		i += 1
												snake_case__          =						tuple(__UpperCamelCase       )
												snake_case__          =						new_word
												if len(__UpperCamelCase       ) == 1:
															break
												else:
															snake_case__          =						get_pairs(__UpperCamelCase       )
									snake_case__          =						""" """.join(__UpperCamelCase       )
									if word == "\n  </w>":
												snake_case__          =						"""\n</w>"""
									snake_case__          =						word
									return word






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						[]
									if self.fix_text is None:
												# Using BERT's BasicTokenizer
												snake_case__          =						self.nlp.tokenize(__UpperCamelCase       )
												for token in text:
															split_tokens.extend(list(self.bpe(__UpperCamelCase       ).split(""" """       )       )       )
									else:
												# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
												snake_case__          =						self.nlp(text_standardize(self.fix_text(__UpperCamelCase       )       )       )
												for token in text:
															split_tokens.extend(list(self.bpe(token.text.lower()       ).split(""" """       )       )       )
									return split_tokens






						def       __lowerCAmelCase(   self						:						List[Any]					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.encoder.get(__UpperCamelCase					,   self.encoder.get(self.unk_token       )       )






						def       __lowerCAmelCase(   self						:						str					,   __UpperCamelCase						:						str       ):



									'''simple docstring'''






									return self.decoder.get(__UpperCamelCase					,   self.unk_token       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]					,   __UpperCamelCase						:						int       ):



									'''simple docstring'''






									snake_case__          =						"""""".join(__UpperCamelCase       ).replace("""</w>"""					,   """ """       ).strip()
									return out_string







						def       __lowerCAmelCase(   self						:						Any					,   __UpperCamelCase						:						str					,   __UpperCamelCase						:						Optional[str] = None       ):



									'''simple docstring'''






									if not os.path.isdir(__UpperCamelCase       ):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
												return
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
									snake_case__          =						os.path.join(
									    __UpperCamelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as f:
												f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=__UpperCamelCase					,   ensure_ascii=__UpperCamelCase       ) + """\n"""       )

									snake_case__          =						0
									with open(__UpperCamelCase					,   """w"""					,   encoding="""utf-8"""       ) as writer:
												writer.write("""#version: 0.2\n"""       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda __UpperCamelCase       : kv[1]       ):
															if index != token_index:
																		logger.warning(
																		    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																		    """ Please check that the tokenizer is not corrupted!"""       )
																		snake_case__          =						token_index
															writer.write(""" """.join(__UpperCamelCase       ) + """\n"""       )
															index += 1

									return vocab_file, merge_file