






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


a 			=							logging.get_logger(__name__)

a 			=							{'vocab_file': 'vocab.txt'}

a 			=							{
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

a 			=							{
    'tau/splinter-base': 512,
    'tau/splinter-base-qass': 512,
    'tau/splinter-large': 512,
    'tau/splinter-large-qass': 512,
}

a 			=							{
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}
class 							__snake_case(       _snake_case   ):
 lowercase			:							str		   =		VOCAB_FILES_NAMES
 lowercase			:							List[str]		   =		PRETRAINED_VOCAB_FILES_MAP
 lowercase			:							Union[str, Any]		   =		PRETRAINED_INIT_CONFIGURATION
 lowercase			:							List[str]		   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase			:							List[Any]		   =		SplinterTokenizer



 def __init__(						self : Any			,		UpperCAmelCase__ : Tuple=None			,		UpperCAmelCase__ : Optional[Any]=None			,		UpperCAmelCase__ : int=True			,		UpperCAmelCase__ : Any="[UNK]"			,		UpperCAmelCase__ : Optional[int]="[SEP]"			,		UpperCAmelCase__ : Tuple="[PAD]"			,		UpperCAmelCase__ : int="[CLS]"			,		UpperCAmelCase__ : str="[MASK]"			,		UpperCAmelCase__ : Optional[int]="[QUESTION]"			,		UpperCAmelCase__ : str=True			,		UpperCAmelCase__ : str=None			,		**UpperCAmelCase__ : List[str]			,		):

    """simple docstring"""






    super().__init__(
        lowercase__			,		tokenizer_file=lowercase__			,		do_lower_case=lowercase__			,		unk_token=lowercase__			,		sep_token=lowercase__			,		pad_token=lowercase__			,		cls_token=lowercase__			,		mask_token=lowercase__			,		tokenize_chinese_chars=lowercase__			,		strip_accents=lowercase__			,		additional_special_tokens=(question_token,)			,		**lowercase__			,		)

    lowercase__    :						List[str]       =			json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
    if (
        pre_tok_state.get("""lowercase"""			,		lowercase__						) != do_lower_case
        or pre_tok_state.get("""strip_accents"""			,		lowercase__						) != strip_accents
    ):
       lowercase__    :						Union[str, Any]       =			getattr(lowercase__			,		pre_tok_state.pop("""type"""						)						)
       lowercase__    :						List[Any]       =			do_lower_case
       lowercase__    :						int       =			strip_accents
       lowercase__    :						str       =			pre_tok_class(**lowercase__						)

    lowercase__    :						List[Any]       =			do_lower_case



 @property
 def     UpperCAmelCase     (						self : str						):

    """simple docstring"""






    return self.convert_tokens_to_ids(self.question_token						)



 def     UpperCAmelCase     (						self : List[Any]			,		UpperCAmelCase__ : List[int]			,		UpperCAmelCase__ : Optional[List[int]] = None						):

    """simple docstring"""






    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    lowercase__    :						List[Any]       =			[self.cls_token_id]
    lowercase__    :						Optional[Any]       =			[self.sep_token_id]
    lowercase__    :						Optional[int]       =			[self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
    if self.padding_side == "right":
       # Input is question-then-context
       return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
    else:
       # Input is context-then-question
       return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



 def     UpperCAmelCase     (						self : Optional[Any]			,		UpperCAmelCase__ : List[int]			,		UpperCAmelCase__ : Optional[List[int]] = None						):

    """simple docstring"""






    lowercase__    :						List[Any]       =			[self.sep_token_id]
    lowercase__    :						int       =			[self.cls_token_id]
    lowercase__    :						Dict       =			[self.question_token_id] + [self.convert_tokens_to_ids("""."""						)]
    if token_ids_a is None:
       return len(cls + token_ids_a + sep						) * [0]

    if self.padding_side == "right":
       # Input is question-then-context
       return len(cls + token_ids_a + question_suffix + sep						) * [0] + len(token_ids_a + sep						) * [1]
    else:
       # Input is context-then-question
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + question_suffix + sep						) * [1]







 def     UpperCAmelCase     (						self : str			,		UpperCAmelCase__ : str			,		UpperCAmelCase__ : Optional[str] = None						):

    """simple docstring"""






    lowercase__    :						List[Any]       =			self._tokenizer.model.save(lowercase__			,		name=lowercase__						)
    return tuple(lowercase__						)







import argparse

import torch

from transformers import RoFormerConfig, RoFormerForMaskedLM, load_tf_weights_in_roformer
from transformers.utils import logging


logging.set_verbosity_info()





def 		__snake_case						(    _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase				):
					"""simple docstring"""






					lowercase						   =       RoFormerConfig.from_json_file(_UpperCAmelCase				)
					print(f"""Building PyTorch model from configuration: {config}"""				)
					lowercase						   =       RoFormerForMaskedLM(_UpperCAmelCase				)

					# Load weights from tf checkpoint
					load_tf_weights_in_roformer(_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase				)

					# Save pytorch-model
					print(f"""Save PyTorch model to {pytorch_dump_path}"""				)
					torch.save(model.state_dict()       ,			_UpperCAmelCase       ,			_use_new_zipfile_serialization=_UpperCAmelCase				)


if __name__ == "__main__":
		__magic_name__       =     argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
		)
		parser.add_argument(
		    '''--bert_config_file''',
		    default=None,
		    type=str,
		    required=True,
		    help=(
		        '''The config json file corresponding to the pre-trained BERT model. \n'''
		        '''This specifies the model architecture.'''
		    ),
		)
		parser.add_argument(
		    '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
		)
		__magic_name__       =     parser.parse_args()
		convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)






