






"""simple docstring"""



from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy


class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						'MarkupLMFeatureExtractor'
    __lowercase    :   Any											=						('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
    __lowercase    :   int											=						True


    def __call__(      self   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase = True   ,			__lowercase = False   ,			__lowercase = None   ,			__lowercase = None   ,			__lowercase = 0   ,			__lowercase = None   ,			__lowercase = None   ,			__lowercase = None   ,			__lowercase = False   ,			__lowercase = False   ,			__lowercase = False   ,			__lowercase = False   ,			__lowercase = True   ,			__lowercase = None   ,			**__lowercase   ,			):
     # first, create nodes and xpaths
     if self.parse_html:
      if html_strings is None:
       raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

      if nodes is not None or xpaths is not None or node_labels is not None:
       raise ValueError(
           """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

      UpperCAmelCase__ 						=				self.feature_extractor(__lowercase			)
      UpperCAmelCase__ 						=				features["""nodes"""]
      UpperCAmelCase__ 						=				features["""xpaths"""]
     else:
      if html_strings is not None:
       raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
      if nodes is None or xpaths is None:
       raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
     if questions is not None and self.parse_html:
      if isinstance(__lowercase   ,			__lowercase			):
       UpperCAmelCase__ 						=				[questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

     UpperCAmelCase__ 						=				self.tokenizer(
         text=questions if questions is not None else nodes   ,			text_pair=nodes if questions is not None else None   ,			xpaths=__lowercase   ,			node_labels=__lowercase   ,			add_special_tokens=__lowercase   ,			padding=__lowercase   ,			truncation=__lowercase   ,			max_length=__lowercase   ,			stride=__lowercase   ,			pad_to_multiple_of=__lowercase   ,			return_token_type_ids=__lowercase   ,			return_attention_mask=__lowercase   ,			return_overflowing_tokens=__lowercase   ,			return_special_tokens_mask=__lowercase   ,			return_offsets_mapping=__lowercase   ,			return_length=__lowercase   ,			verbose=__lowercase   ,			return_tensors=__lowercase   ,			**__lowercase   ,			)

     return encoded_inputs


    def 		A__      (      self   ,			*__lowercase   ,			**__lowercase			):
     return self.tokenizer.batch_decode(*__lowercase   ,			**__lowercase			)


    def 		A__      (      self   ,			*__lowercase   ,			**__lowercase			):
     return self.tokenizer.decode(*__lowercase   ,			**__lowercase			)






    @property
    def 		A__      (      self			):
     UpperCAmelCase__ 						=				self.tokenizer.model_input_names
     return tokenizer_input_names











"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


a  :						List[Any]					       =							logging.get_logger(__name__)

a  :						List[Any]					       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a  :						Any					       =							{
    '''vocab_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''microsoft/mpnet-base''': '''https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json''',
    },
}

a  :						str					       =							{
    '''microsoft/mpnet-base''': 512,
}

a  :						Union[str, Any]					       =							{
    '''microsoft/mpnet-base''': {'''do_lower_case''': True},
}


class 	_UpperCamelCase ( __UpperCamelCase							):



    '''simple docstring'''


    __lowercase    :   Optional[Any]											=						VOCAB_FILES_NAMES
    __lowercase    :   Optional[int]											=						PRETRAINED_VOCAB_FILES_MAP
    __lowercase    :   Dict											=						PRETRAINED_INIT_CONFIGURATION
    __lowercase    :   Optional[Any]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowercase    :   Dict											=						MPNetTokenizer
    __lowercase    :   Dict											=						['input_ids', 'attention_mask']


    def __init__(      self   ,			__lowercase=None   ,			__lowercase=None   ,			__lowercase=True   ,			__lowercase="<s>"   ,			__lowercase="</s>"   ,			__lowercase="</s>"   ,			__lowercase="<s>"   ,			__lowercase="[UNK]"   ,			__lowercase="<pad>"   ,			__lowercase="<mask>"   ,			__lowercase=True   ,			__lowercase=None   ,			**__lowercase   ,			):
     super().__init__(
         __lowercase   ,			tokenizer_file=__lowercase   ,			do_lower_case=__lowercase   ,			bos_token=__lowercase   ,			eos_token=__lowercase   ,			sep_token=__lowercase   ,			cls_token=__lowercase   ,			unk_token=__lowercase   ,			pad_token=__lowercase   ,			mask_token=__lowercase   ,			tokenize_chinese_chars=__lowercase   ,			strip_accents=__lowercase   ,			**__lowercase   ,			)

     UpperCAmelCase__ 						=				json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
     if (
         pre_tok_state.get("""lowercase"""   ,			__lowercase			) != do_lower_case
         or pre_tok_state.get("""strip_accents"""   ,			__lowercase			) != strip_accents
     ):
      UpperCAmelCase__ 						=				getattr(__lowercase   ,			pre_tok_state.pop("""type"""			)			)
      UpperCAmelCase__ 						=				do_lower_case
      UpperCAmelCase__ 						=				strip_accents
      UpperCAmelCase__ 						=				pre_tok_class(**__lowercase			)

     UpperCAmelCase__ 						=				do_lower_case


    @property
    def 		A__      (      self			):
     if self._mask_token is None:
      if self.verbose:
       logger.error("""Using mask_token, but it is not set yet."""			)
      return None
     return str(self._mask_token			)


    @mask_token.setter
    def 		A__      (      self   ,			__lowercase			):
     UpperCAmelCase__ 						=				AddedToken(__lowercase   ,			lstrip=__lowercase   ,			rstrip=__lowercase			) if isinstance(__lowercase   ,			__lowercase			) else value
     UpperCAmelCase__ 						=				value


    def 		A__      (      self   ,			__lowercase   ,			__lowercase=None			):
     UpperCAmelCase__ 						=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
      return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				[self.sep_token_id]
     UpperCAmelCase__ 						=				[self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]






    def 		A__      (      self   ,			__lowercase   ,			__lowercase = None			):
     UpperCAmelCase__ 						=				self._tokenizer.model.save(__lowercase   ,			name=__lowercase			)
     return tuple(__lowercase			)



