






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
					import torch

snake_case_   :	Optional[Any]							      =logging.get_logger(__name__)



def      UpperCAmelCase						( ):


		'''simple docstring'''

		return importlib.util.find_spec("deepspeed"	) is not None


if is_accelerate_available() and is_deepspeed_available():
					from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
					# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
					# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
					from builtins import object as DeepSpeedConfig





class        a__      (	lowerCAmelCase__		):






		def __init__(      self							, lowercase__						)				->				Tuple:
				# set global weakref object
				set_hf_deepspeed_config(self						)
				dep_version_check("accelerate"						)
				dep_version_check("deepspeed"						)
				super().__init__(lowercase__						)





class        a__      (	lowerCAmelCase__		):






		def __init__(      self							, lowercase__						)				->				Union[str, Any]:
				super().__init__(lowercase__						)
				__A    =							None
				__A    =							[]






		def 					_lowerCamelCase					(      self						)				->				Union[str, Any]:
				if self._dtype is None:
						raise ValueError("trainer_config_process() wasn't called yet to tell dtype"						)
				return self._dtype






		def 					_lowerCamelCase					(      self							, lowercase__						)				->				str:
				__A    =							self.get_value(lowercase__						)
				if val is None:
						return False
				else:
						return val == "auto"






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__=None							, lowercase__=True						)				->				Dict:
				__A ,   __A    =							self.find_config_node(lowercase__						)
				if config is None:
						return

				if config.get(lowercase__						) == "auto":
						__A    =							hf_val
						return

				if not must_match:
						return

				__A    =							config.get(lowercase__						)
				if ds_val is not None and ds_val != hf_val:
						self.mismatches.append(F"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""						)

		UpperCAmelCase_							:       int 					= partialmethod(lowerCAmelCase__	,       must_match=lowerCAmelCase__		)






		def 					_lowerCamelCase					(      self							, lowercase__						)				->				int:
				__A    =							args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
				self.fill_match(
				    "train_micro_batch_size_per_gpu"							, args.per_device_train_batch_size							, "per_device_train_batch_size"						)
				self.fill_match("gradient_accumulation_steps"							, args.gradient_accumulation_steps							, "gradient_accumulation_steps"						)
				self.fill_match("train_batch_size"							, lowercase__							, "train_batch_size (calculated)"						)
				self.fill_match("gradient_clipping"							, args.max_grad_norm							, "max_grad_norm"						)

				self.fill_match("optimizer.params.lr"							, args.learning_rate							, "learning_rate"						)
				self.fill_match("optimizer.params.betas"							, [args.adam_betaa, args.adam_betaa]							, "adam_beta1+adam_beta2"						)
				self.fill_match("optimizer.params.eps"							, args.adam_epsilon							, "adam_epsilon"						)
				self.fill_match("optimizer.params.weight_decay"							, args.weight_decay							, "weight_decay"						)

				self.fill_only("scheduler.params.warmup_min_lr"							, 0						)  # not a trainer arg
				self.fill_match("scheduler.params.warmup_max_lr"							, args.learning_rate							, "learning_rate"						)
				# total_num_steps - will get set in trainer_config_finalize

				# fp16
				if args.fpaa or args.fpaa_full_eval:
						__A    =							"apex" if args.fpaa_backend == "apex" else "amp"
				else:
						__A    =							None

				if args.save_on_each_node:
						# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
						__A    =							self.config.get("checkpoint"							, {}						)
						__A    =							args.save_on_each_node

				# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
				# any here unless the user did the work
				self.fill_match(
				    "fp16.enabled"							, ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")							, "fp16|fp16_full_eval+fp16_backend(amp)"							, )

				# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
				# ZeRO features
				self.fill_match("amp.enabled"							, fpaa_backend == "apex"							, "fp16+fp16_backend(apex)"						)
				self.fill_match("amp.opt_level"							, args.fpaa_opt_level							, "fp16_opt_level"						)

				self.fill_match("bf16.enabled"							, (args.bfaa or args.bfaa_full_eval)							, "bf16|bf16_full_eval"						)

				# deepspeed's default mode is fp16 unless there is a config that says differently
				if self.is_true("bf16.enabled"						):
						__A    =							torch.bfloataa
				elif self.is_false("fp16.enabled"						):
						__A    =							torch.floataa
				else:
						__A    =							torch.floataa






		def 					_lowerCamelCase					(      self							, lowercase__							, lowercase__							, lowercase__						)				->				Dict:
				__A    =							[
				    "zero_optimization.reduce_bucket_size",
				    "zero_optimization.stage3_prefetch_bucket_size",
				    "zero_optimization.stage3_param_persistence_threshold",
				]
				__A    =							[x for x in hidden_size_based_keys if self.is_auto(lowercase__						)]

				if len(lowercase__						) > 0:
						if hasattr(model.config							, "hidden_size"						):
								__A    =							model.config.hidden_size
						elif hasattr(model.config							, "hidden_sizes"						):
								# if there are many hidden sizes pick the largest one
								__A    =							max(model.config.hidden_sizes						)
						else:
								raise ValueError(
								    "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, "
								    "therefore it's not possible to automatically fill out the following `auto` entries "
								    F"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
								    "`auto` values for these keys with an integer value of your choice."						)

						self.fill_only("zero_optimization.reduce_bucket_size"							, hidden_size * hidden_size						)
						if self.is_zeroa():
								# automatically assign the optimal config values based on model config
								self.fill_only("zero_optimization.stage3_prefetch_bucket_size"							, 0.9 * hidden_size * hidden_size						)
								self.fill_only("zero_optimization.stage3_param_persistence_threshold"							, 10 * hidden_size						)

        # scheduler
				self.fill_match("scheduler.params.total_num_steps"							, lowercase__							, "num_training_steps (calculated)"						)
				self.fill_match("scheduler.params.warmup_num_steps"							, args.get_warmup_steps(lowercase__						)							, "warmup_steps"						)

				if len(self.mismatches						) > 0:
						__A    =							"\n".join(self.mismatches						)
						raise ValueError(
						    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
						    F""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."""						)



snake_case_   :	Tuple							      =None



def      UpperCAmelCase						( lowerCAmelCase__	):


		'''simple docstring'''

		global _hf_deepspeed_config_weak_ref
		# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
		__A    =							weakref.ref(lowerCAmelCase__	)



def      UpperCAmelCase						( ):


		'''simple docstring'''

		global _hf_deepspeed_config_weak_ref
		__A    =							None



def      UpperCAmelCase						( ):


		'''simple docstring'''

		if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
				return _hf_deepspeed_config_weak_ref().is_zeroa()
		else:
				return False



def      UpperCAmelCase						( ):


		'''simple docstring'''

		if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
				return _hf_deepspeed_config_weak_ref().config
		else:
				return None



def      UpperCAmelCase						( lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__	):


		'''simple docstring'''

		from accelerate.utils import DummyOptim, DummyScheduler

		__A    =							hf_deepspeed_config.config

		# Optimizer + Scheduler
		# Currently supported combos:
		# 1. DS scheduler + DS optimizer: Yes
		# 2. HF scheduler + HF optimizer: Yes
		# 3. DS scheduler + HF optimizer: Yes
		# 4. HF scheduler + DS optimizer: No
		#
		# Unless Offload is enabled in which case it's:
		# 1. DS scheduler + DS optimizer: Yes
		# 2. HF scheduler + HF optimizer: Mostly*
		# 3. DS scheduler + HF optimizer: Mostly*
		# 4. HF scheduler + DS optimizer: No
		#
		# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

		__A    =							None
		if "optimizer" in config:
				if args.adafactor:
						raise ValueError(
						    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
						    "Only one optimizer can be configured."	)
				__A    =							DummyOptim(params=lowerCAmelCase__	)
		else:
				if hf_deepspeed_config.is_offload():
						logger.info(
						    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
						    " custom optimizer has both CPU and GPU implementation (except LAMB)"	)

				# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
				# But trainer uses AdamW by default.
				__A    =							trainer.create_optimizer()
				# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
				__A    =							True

		__A    =							None
		if "scheduler" in config:
				__A    =							DummyScheduler(lowerCAmelCase__	)
		else:
				if isinstance(lowerCAmelCase__					,     lowerCAmelCase__	):
						raise ValueError(
						    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
						    "Please configure a scheduler in the DeepSpeed config."	)
				__A    =							trainer.create_scheduler(num_training_steps=lowerCAmelCase__					,     optimizer=lowerCAmelCase__	)

		return optimizer, lr_scheduler



def      UpperCAmelCase						( lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__=False	):


		'''simple docstring'''

		from deepspeed.utils import logger as ds_logger

		__A    =							trainer.model
		__A    =							trainer.args

		__A    =							trainer.accelerator.state.deepspeed_plugin.hf_ds_config

		# resume config update - some bits like `model` and `num_training_steps` only become available during train
		hf_deepspeed_config.trainer_config_finalize(lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__	)

		# set the Deepspeed log level consistent with the Trainer
		ds_logger.setLevel(args.get_process_log_level()	)

		if inference:
				# only Z3 makes sense for the inference
				if not hf_deepspeed_config.is_zeroa():
						raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"	)

				# in case the training config is re-used for inference
				hf_deepspeed_config.del_config_sub_tree("optimizer"	)
				hf_deepspeed_config.del_config_sub_tree("lr_scheduler"	)
				__A ,   __A    =							None, None
				__A    =							None
		else:
				__A    =							None  # important for when deepspeed_init is used as re-init
				__A    =							list(filter(lambda lowerCAmelCase__	: p.requires_grad					,     model.parameters()	)	)
				__A ,   __A    =							deepspeed_optim_sched(
				    lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__					,     lowerCAmelCase__	)

		# keep for quick debug:
		# from pprint import pprint; pprint(config)

		return optimizer, lr_scheduler





def      UpperCAmelCase						( lowerCAmelCase__					,     lowerCAmelCase__	):


		'''simple docstring'''

		import glob

		__A    =							sorted(glob.glob(F"""{checkpoint_path}/global_step*"""	)	)

		if len(lowerCAmelCase__	) > 0:
				logger.info(F"""Attempting to resume from {checkpoint_path}"""	)
				# this magically updates self.optimizer and self.lr_scheduler
				__A ,   __A    =							deepspeed_engine.load_checkpoint(
				    lowerCAmelCase__					,     load_optimizer_states=lowerCAmelCase__					,     load_lr_scheduler_states=lowerCAmelCase__	)
				if load_path is None:
						raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}"""	)
		else:
				raise ValueError(F"""Can't find a valid checkpoint at {checkpoint_path}"""	)









import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowerCAmelCase_     =			logging.get_logger(__name__)

lowerCAmelCase_     =			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase_     =			{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

lowerCAmelCase_     =			{
    'openai-gpt': 512,
}







def   snake_case		(				UpperCAmelCase						:							Union[str, Any] ):
				A									=   text.strip()
				if not text:
								return []
				A									=   text.split()
				return tokens
class   UpperCamelCase			(					snake_case__			):







		"""simple docstring"""



		def __init__(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]=True    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]=None    ,_SCREAMING_SNAKE_CASE		:							int=True    ,_SCREAMING_SNAKE_CASE		:							Any=None    ,_SCREAMING_SNAKE_CASE		:							str=True    ,)     -> List[Any]:


						'''simple docstring'''
						if never_split is None:
										A									=   []
						A									=   do_lower_case
						A									=   set(_SCREAMING_SNAKE_CASE		)
						A									=   tokenize_chinese_chars
						A									=   strip_accents
						A									=   do_split_on_punc
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							int    ,_SCREAMING_SNAKE_CASE		:							Optional[int]=None		)     -> int:


						'''simple docstring'''
						A									=   self.never_split.union(set(_SCREAMING_SNAKE_CASE		)		) if never_split else self.never_split
						A									=   self._clean_text(_SCREAMING_SNAKE_CASE		)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
										A									=   self._tokenize_chinese_chars(_SCREAMING_SNAKE_CASE		)
						# prevents treating the same character with different unicode codepoints as different characters
						A									=   unicodedata.normalize('NFC'    ,_SCREAMING_SNAKE_CASE		)
						A									=   whitespace_tokenize(_SCREAMING_SNAKE_CASE		)
						A									=   []
						for token in orig_tokens:
										if token not in never_split:
														if self.do_lower_case:
																		A									=   token.lower()
																		if self.strip_accents is not False:
																						A									=   self._run_strip_accents(_SCREAMING_SNAKE_CASE		)
														elif self.strip_accents:
																		A									=   self._run_strip_accents(_SCREAMING_SNAKE_CASE		)
										split_tokens.extend(self._run_split_on_punc(_SCREAMING_SNAKE_CASE    ,_SCREAMING_SNAKE_CASE		)		)

						A									=   whitespace_tokenize(' '.join(_SCREAMING_SNAKE_CASE		)		)
						return output_tokens
		def    A(	self		:							int    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Optional[Any]:


						'''simple docstring'''
						A									=   unicodedata.normalize('NFD'    ,_SCREAMING_SNAKE_CASE		)
						A									=   []
						for char in text:
										A									=   unicodedata.category(_SCREAMING_SNAKE_CASE		)
										if cat == "Mn":
														continue
										output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)
		def    A(	self		:							Tuple    ,_SCREAMING_SNAKE_CASE		:							List[str]    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]=None		)     -> int:


						'''simple docstring'''
						if not self.do_split_on_punc or (never_split is not None and text in never_split):
										return [text]
						A									=   list(_SCREAMING_SNAKE_CASE		)
						A									=   0
						A									=   True
						A									=   []
						while i < len(_SCREAMING_SNAKE_CASE		):
										A									=   chars[i]
										if _is_punctuation(_SCREAMING_SNAKE_CASE		):
														output.append([char]		)
														A									=   True
										else:
														if start_new_word:
																		output.append([]		)
														A									=   False
														output[-1].append(_SCREAMING_SNAKE_CASE		)
										i += 1

						return ["".join(_SCREAMING_SNAKE_CASE		) for x in output]
		def    A(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> Optional[int]:


						'''simple docstring'''
						A									=   []
						for char in text:
										A									=   ord(_SCREAMING_SNAKE_CASE		)
										if self._is_chinese_char(_SCREAMING_SNAKE_CASE		):
														output.append(' '		)
														output.append(_SCREAMING_SNAKE_CASE		)
														output.append(' '		)
										else:
														output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)
		def    A(	self		:							Dict    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Tuple:


						'''simple docstring'''
						# This defines a "chinese character" as anything in the CJK Unicode block:
						#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
						#
						# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
						# despite its name. The modern Korean Hangul alphabet is a different block,
						# as is Japanese Hiragana and Katakana. Those alphabets are used to write
						# space-separated words, so they are not treated specially and handled
						# like the all of the other languages.
						if (
						    (cp >= 0x4e00 and cp <= 0x9fff)
						    or (cp >= 0x3400 and cp <= 0x4dbf)  #
						    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
						    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
						    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
						    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
						    or (cp >= 0xf900 and cp <= 0xfaff)
						    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
						):  #
										return True

						return False







		def    A(	self		:							Union[str, Any]    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> List[str]:


						'''simple docstring'''
						A									=   []
						for char in text:
										A									=   ord(_SCREAMING_SNAKE_CASE		)
										if cp == 0 or cp == 0xfffd or _is_control(_SCREAMING_SNAKE_CASE		):
														continue
										if _is_whitespace(_SCREAMING_SNAKE_CASE		):
														output.append(' '		)
										else:
														output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)







def   snake_case		(				UpperCAmelCase						:							str ):
				A									=   set()
				A									=   word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								A									=   char
				return pairs







def   snake_case		(				UpperCAmelCase						:							str ):
				A									=   text.replace('—',    '-' )
				A									=   text.replace('–',    '-' )
				A									=   text.replace('―',    '-' )
				A									=   text.replace('…',    '...' )
				A									=   text.replace('´',    '\'' )
				A									=   re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)',    r' \1 ',    UpperCAmelCase )
				A									=   re.sub(r'\s*\n\s*',    ' \n ',    UpperCAmelCase )
				A									=   re.sub(r'[^\S\n]+',    ' ',    UpperCAmelCase )
				return text.strip()



class   UpperCamelCase			(					snake_case__			):







		"""simple docstring"""



		snake_case			 = VOCAB_FILES_NAMES
		snake_case			 = PRETRAINED_VOCAB_FILES_MAP
		snake_case			 = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case			 = ["input_ids", "attention_mask"]
		def __init__(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]    ,_SCREAMING_SNAKE_CASE		:							Dict    ,_SCREAMING_SNAKE_CASE		:							Dict="<unk>"    ,**_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Dict:


						'''simple docstring'''
						super().__init__(unk_token=_SCREAMING_SNAKE_CASE    ,**_SCREAMING_SNAKE_CASE		)

						try:
										import ftfy
										from spacy.lang.en import English

										A									=   English()
										A									=   _nlp.tokenizer
										A									=   ftfy.fix_text
						except ImportError:
										logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'		)
										A									=   BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		)
										A									=   None

						with open(_SCREAMING_SNAKE_CASE    ,encoding='utf-8'		) as vocab_handle:
										A									=   json.load(_SCREAMING_SNAKE_CASE		)
						A									=   {v: k for k, v in self.encoder.items()}
						with open(_SCREAMING_SNAKE_CASE    ,encoding='utf-8'		) as merges_handle:
										A									=   merges_handle.read().split('\n'		)[1:-1]
						A									=   [tuple(merge.split()		) for merge in merges]
						A									=   dict(zip(_SCREAMING_SNAKE_CASE    ,range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
						A									=   {}
		@property
		def    A(	self		:							Union[str, Any]		)     -> Dict:


						'''simple docstring'''
						return True
		@property
		def    A(	self		:							Optional[int]		)     -> str:


						'''simple docstring'''
						return len(self.encoder		)
		def    A(	self		:							Optional[Any]		)     -> str:


						'''simple docstring'''
						return dict(self.encoder    ,**self.added_tokens_encoder		)
		def    A(	self		:							Optional[int]    ,_SCREAMING_SNAKE_CASE		:							Dict		)     -> List[Any]:


						'''simple docstring'''
						A									=   tuple(token[:-1]		) + (token[-1] + '</w>',)
						if token in self.cache:
										return self.cache[token]
						A									=   get_pairs(_SCREAMING_SNAKE_CASE		)

						if not pairs:
										return token + "</w>"

						while True:
										A									=   min(_SCREAMING_SNAKE_CASE    ,key=lambda _SCREAMING_SNAKE_CASE		: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE    ,float('inf'		)		)		)
										if bigram not in self.bpe_ranks:
														break
										A			,		A									=   bigram
										A									=   []
										A									=   0
										while i < len(_SCREAMING_SNAKE_CASE		):
														try:
																		A									=   word.index(_SCREAMING_SNAKE_CASE    ,_SCREAMING_SNAKE_CASE		)
														except ValueError:
																		new_word.extend(word[i:]		)
																		break
														else:
																		new_word.extend(word[i:j]		)
																		A									=   j

														if word[i] == first and i < len(_SCREAMING_SNAKE_CASE		) - 1 and word[i + 1] == second:
																		new_word.append(first + second		)
																		i += 2
														else:
																		new_word.append(word[i]		)
																		i += 1
										A									=   tuple(_SCREAMING_SNAKE_CASE		)
										A									=   new_word
										if len(_SCREAMING_SNAKE_CASE		) == 1:
														break
										else:
														A									=   get_pairs(_SCREAMING_SNAKE_CASE		)
						A									=   ' '.join(_SCREAMING_SNAKE_CASE		)
						if word == "\n  </w>":
										A									=   '\n</w>'
						A									=   word
						return word
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							Optional[int]		)     -> List[Any]:


						'''simple docstring'''
						A									=   []
						if self.fix_text is None:
										# Using BERT's BasicTokenizer
										A									=   self.nlp.tokenize(_SCREAMING_SNAKE_CASE		)
										for token in text:
														split_tokens.extend(list(self.bpe(_SCREAMING_SNAKE_CASE		).split(' '		)		)		)
						else:
										# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
										A									=   self.nlp(text_standardize(self.fix_text(_SCREAMING_SNAKE_CASE		)		)		)
										for token in text:
														split_tokens.extend(list(self.bpe(token.text.lower()		).split(' '		)		)		)
						return split_tokens
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							List[Any]		)     -> Any:


						'''simple docstring'''
						return self.encoder.get(_SCREAMING_SNAKE_CASE    ,self.encoder.get(self.unk_token		)		)
		def    A(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> List[str]:


						'''simple docstring'''
						return self.decoder.get(_SCREAMING_SNAKE_CASE    ,self.unk_token		)
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							str		)     -> Any:


						'''simple docstring'''
						A									=   ''.join(_SCREAMING_SNAKE_CASE		).replace('</w>'    ,' '		).strip()
						return out_string







		def    A(	self		:							Tuple    ,_SCREAMING_SNAKE_CASE		:							str    ,_SCREAMING_SNAKE_CASE		:							Optional[str] = None		)     -> Tuple[str]:


						'''simple docstring'''
						if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
										return
						A									=   os.path.join(
						    _SCREAMING_SNAKE_CASE    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
						A									=   os.path.join(
						    _SCREAMING_SNAKE_CASE    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']		)

						with open(_SCREAMING_SNAKE_CASE    ,'w'    ,encoding='utf-8'		) as f:
										f.write(json.dumps(self.encoder    ,indent=2    ,sort_keys=_SCREAMING_SNAKE_CASE    ,ensure_ascii=_SCREAMING_SNAKE_CASE		) + '\n'		)

						A									=   0
						with open(_SCREAMING_SNAKE_CASE    ,'w'    ,encoding='utf-8'		) as writer:
										writer.write('#version: 0.2\n'		)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,key=lambda _SCREAMING_SNAKE_CASE		: kv[1]		):
														if index != token_index:
																		logger.warning(
																		    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																		    ' Please check that the tokenizer is not corrupted!'		)
																		A									=   token_index
														writer.write(' '.join(_SCREAMING_SNAKE_CASE		) + '\n'		)
														index += 1

						return vocab_file, merge_file





