





import json
import os
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A				:    int       =   logging.get_logger(__name__)

__A				:    Optional[Any]       =   {"""vocab_file""": """vocab.json"""}

__A				:    Optional[Any]       =   {
    """vocab_file""": {
        """mgp-str""": """https://huggingface.co/alibaba-damo/mgp-str-base/blob/main/vocab.json""",
    }
}

__A				:    int       =   {"""mgp-str""": 27}

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	def __init__(    self							,					_A							,					_A="[GO]"							,					_A="[GO]"							,					_A="[s]"							,					_A="[GO]"							,					**_A		):






				'''simple docstring'''







				super().__init__(
				    unk_token=_A							,					bos_token=_A							,					eos_token=_A							,					pad_token=_A							,					**_A							,					)

				with open(_A							,					encoding='''utf-8'''		) as vocab_handle:
							UpperCAmelCase   		=				json.load(_A		)
				UpperCAmelCase   		=				{v: k for k, v in self.vocab.items()}
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				return len(self.vocab		)
	def      _lowercase				(    self		):






				'''simple docstring'''







				return dict(self.vocab							,					**self.added_tokens_encoder		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				for s in text:
							char_tokens.extend(_A		)
				return char_tokens
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.vocab.get(_A							,					self.vocab.get(self.unk_token		)		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.decoder.get(_A		)







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				if not os.path.isdir(_A		):
							logger.error('''Vocabulary path ({}) should be a directory'''.format(_A		)		)
							return
				UpperCAmelCase   		=				os.path.join(
				    _A							,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

				with open(_A							,					'''w'''							,					encoding='''utf-8'''		) as f:
							f.write(json.dumps(self.vocab							,					indent=2							,					sort_keys=_A							,					ensure_ascii=_A		) + '''\n'''		)

				return (vocab_file,)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase  :					str						    =							{
    """configuration_funnel""": ["""FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """FunnelConfig"""],
    """convert_funnel_original_tf_checkpoint_to_pytorch""": [],
    """tokenization_funnel""": ["""FunnelTokenizer"""],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase  :					Tuple						    =							["""FunnelTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase  :					List[str]						    =							[
      """FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """FunnelBaseModel""",
      """FunnelForMaskedLM""",
      """FunnelForMultipleChoice""",
      """FunnelForPreTraining""",
      """FunnelForQuestionAnswering""",
      """FunnelForSequenceClassification""",
      """FunnelForTokenClassification""",
      """FunnelModel""",
      """FunnelPreTrainedModel""",
      """load_tf_weights_in_funnel""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase  :					Optional[Any]						    =							[
      """TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFFunnelBaseModel""",
      """TFFunnelForMaskedLM""",
      """TFFunnelForMultipleChoice""",
      """TFFunnelForPreTraining""",
      """TFFunnelForQuestionAnswering""",
      """TFFunnelForSequenceClassification""",
      """TFFunnelForTokenClassification""",
      """TFFunnelModel""",
      """TFFunnelPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
  from .tokenization_funnel import FunnelTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_funnel_fast import FunnelTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_funnel import (
        FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
        FunnelBaseModel,
        FunnelForMaskedLM,
        FunnelForMultipleChoice,
        FunnelForPreTraining,
        FunnelForQuestionAnswering,
        FunnelForSequenceClassification,
        FunnelForTokenClassification,
        FunnelModel,
        FunnelPreTrainedModel,
        load_tf_weights_in_funnel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_funnel import (
        TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFFunnelBaseModel,
        TFFunnelForMaskedLM,
        TFFunnelForMultipleChoice,
        TFFunnelForPreTraining,
        TFFunnelForQuestionAnswering,
        TFFunnelForSequenceClassification,
        TFFunnelForTokenClassification,
        TFFunnelModel,
        TFFunnelPreTrainedModel,
    )

else:
  import sys

  lowercase  :					Any						    =							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
