



from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


UpperCAmelCase							=       logging.get_logger(__name__)

UpperCAmelCase							=       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase							=       {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

UpperCAmelCase							=       {
    """openai-gpt""": 512,
}

class 			__snake_case   ( SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							UpperCamelCase__					:   Union[str, Any]       =       VOCAB_FILES_NAMES
							UpperCamelCase__					:   Optional[Any]       =       PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__					:   Dict       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__					:   Any       =       ["""input_ids""", """attention_mask"""]
							UpperCamelCase__					:   Optional[Any]       =       OpenAIGPTTokenizer


							def __init__(				self	,	a_=None	,	a_=None	,	a_=None	,	a_="<unk>"	,	**a_		):
											super().__init__(a_	,	a_	,	tokenizer_file=a_	,	unk_token=a_	,	**a_		)


							@property
							def _a   (				self		):
											return True







							def _a   (				self	,	a_	,	a_ = None		):
											a__					   =		self._tokenizer.model.save(a_	,	name=a_		)
											return tuple(a_		)



'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


__lowercase	=    {
    '''configuration_xlm_prophetnet''': ['''XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLMProphetNetConfig'''],
}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowercase	=    ['''XLMProphetNetTokenizer''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowercase	=    [
							    '''XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''XLMProphetNetDecoder''',
							    '''XLMProphetNetEncoder''',
							    '''XLMProphetNetForCausalLM''',
							    '''XLMProphetNetForConditionalGeneration''',
							    '''XLMProphetNetModel''',
							    '''XLMProphetNetPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xlm_prophetnet import (
														    XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
														    XLMProphetNetDecoder,
														    XLMProphetNetEncoder,
														    XLMProphetNetForCausalLM,
														    XLMProphetNetForConditionalGeneration,
														    XLMProphetNetModel,
														    XLMProphetNetPreTrainedModel,
														)

else:
							import sys

							__lowercase	=    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






