




'''simple docstring'''





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a		:	str       =      logging.get_logger(__name__)


a		:	List[Any]       =      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
a		:	List[Any]       =      {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
}

a		:	Dict       =      {
    '''RUCAIBox/mvp''': 1_0_2_4,
}





@lru_cache()
def UpperCAmelCase    (       ):
    _a				:							str							       =      (
        list(range(ord("""!"""						)   ,	ord("""~"""						) + 1						)						) + list(range(ord("""¡"""						)   ,	ord("""¬"""						) + 1						)						) + list(range(ord("""®"""						)   ,	ord("""ÿ"""						) + 1						)						)
    )
    _a				:							str							       =      bs[:]
    _a				:							Dict							       =      0
    for b in range(2**8						):
        if b not in bs:
            bs.append(UpperCAmelCase_						)
            cs.append(2**8 + n						)
            n += 1
    _a				:							Tuple							       =      [chr(UpperCAmelCase_						) for n in cs]
    return dict(zip(UpperCAmelCase_   ,	UpperCAmelCase_						)						)





def UpperCAmelCase    (       UpperCAmelCase_						):
    _a				:							str							       =      set()
    _a				:							Any							       =      word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        _a				:							List[str]							       =      char
    return pairs





class      UpperCamelCase       (	snake_case__       ):
  a     :Union[str, Any]          =					VOCAB_FILES_NAMES
  a     :Union[str, Any]          =					PRETRAINED_VOCAB_FILES_MAP
  a     :Union[str, Any]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a     :Optional[Any]          =					['input_ids', 'attention_mask']



  def __init__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase="replace"		,     __lowerCAmelCase="<s>"		,     __lowerCAmelCase="</s>"		,     __lowerCAmelCase="</s>"		,     __lowerCAmelCase="<s>"		,     __lowerCAmelCase="<unk>"		,     __lowerCAmelCase="<pad>"		,     __lowerCAmelCase="<mask>"		,     __lowerCAmelCase=False		,     **__lowerCAmelCase		,     ):







      '''simple docstring'''


      _a				:							int							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else bos_token
      _a				:							Tuple							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else eos_token
      _a				:							Tuple							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else sep_token
      _a				:							Tuple							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else cls_token
      _a				:							int							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else unk_token
      _a				:							List[Any]							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      _a				:							Dict							       =      AddedToken(__lowerCAmelCase		,     lstrip=__lowerCAmelCase		,     rstrip=__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else mask_token

      super().__init__(
          errors=__lowerCAmelCase		,     bos_token=__lowerCAmelCase		,     eos_token=__lowerCAmelCase		,     unk_token=__lowerCAmelCase		,     sep_token=__lowerCAmelCase		,     cls_token=__lowerCAmelCase		,     pad_token=__lowerCAmelCase		,     mask_token=__lowerCAmelCase		,     add_prefix_space=__lowerCAmelCase		,     **__lowerCAmelCase		,     )
      with open(__lowerCAmelCase		,     encoding="""utf-8""" ) as vocab_handle:
          _a				:							Optional[Any]							       =      json.load(__lowerCAmelCase )
      _a				:							int							       =      {v: k for k, v in self.encoder.items()}
      _a				:							Union[str, Any]							       =      errors  # how to handle errors in decoding
      _a				:							List[str]							       =      bytes_to_unicode()
      _a				:							Optional[Any]							       =      {v: k for k, v in self.byte_encoder.items()}
      with open(__lowerCAmelCase		,     encoding="""utf-8""" ) as merges_handle:
          _a				:							List[Any]							       =      merges_handle.read().split("""\n""" )[1:-1]
      _a				:							Tuple							       =      [tuple(merge.split() ) for merge in bpe_merges]
      _a				:							Tuple							       =      dict(zip(__lowerCAmelCase		,     range(len(__lowerCAmelCase ) ) ) )
      _a				:							Optional[Any]							       =      {}
      _a				:							Tuple							       =      add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      _a				:							List[str]							       =      re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )



  @property
  def     a_							(			self ):







      '''simple docstring'''


      return len(self.encoder )



  def     a_							(			self ):







      '''simple docstring'''


      return dict(self.encoder		,     **self.added_tokens_encoder )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      if token in self.cache:
          return self.cache[token]
      _a				:							List[Any]							       =      tuple(__lowerCAmelCase )
      _a				:							Any							       =      get_pairs(__lowerCAmelCase )

      if not pairs:
          return token

      while True:
          _a				:							List[str]							       =      min(__lowerCAmelCase		,     key=lambda __lowerCAmelCase : self.bpe_ranks.get(__lowerCAmelCase		,     float("""inf""" ) ) )
          if bigram not in self.bpe_ranks:
              break
          _a       , _a				:							Optional[int]							       =      bigram
          _a				:							int							       =      []
          _a				:							List[Any]							       =      0
          while i < len(__lowerCAmelCase ):
              try:
                  _a				:							Optional[int]							       =      word.index(__lowerCAmelCase		,     __lowerCAmelCase )
              except ValueError:
                  new_word.extend(word[i:] )
                  break
              else:
                  new_word.extend(word[i:j] )
                  _a				:							Any							       =      j

              if word[i] == first and i < len(__lowerCAmelCase ) - 1 and word[i + 1] == second:
                  new_word.append(first + second )
                  i += 2
              else:
                  new_word.append(word[i] )
                  i += 1
          _a				:							List[str]							       =      tuple(__lowerCAmelCase )
          _a				:							List[str]							       =      new_word
          if len(__lowerCAmelCase ) == 1:
              break
          else:
              _a				:							Union[str, Any]							       =      get_pairs(__lowerCAmelCase )
      _a				:							Dict							       =      """ """.join(__lowerCAmelCase )
      _a				:							int							       =      word
      return word



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Optional[Any]							       =      []
      for token in re.findall(self.pat		,     __lowerCAmelCase ):
          _a				:							Any							       =      """""".join(
              self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase ).split(""" """ ) )
      return bpe_tokens



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      return self.encoder.get(__lowerCAmelCase		,     self.encoder.get(self.unk_token ) )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      return self.decoder.get(__lowerCAmelCase )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							str							       =      """""".join(__lowerCAmelCase )
      _a				:							List[Any]							       =      bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""		,     errors=self.errors )
      return text



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None ):







      '''simple docstring'''


      if not os.path.isdir(__lowerCAmelCase ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
          return
      _a				:							Optional[int]							       =      os.path.join(
          __lowerCAmelCase		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
      _a				:							Optional[int]							       =      os.path.join(
          __lowerCAmelCase		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

      with open(__lowerCAmelCase		,     """w"""		,     encoding="""utf-8""" ) as f:
          f.write(json.dumps(self.encoder		,     indent=2		,     sort_keys=__lowerCAmelCase		,     ensure_ascii=__lowerCAmelCase ) + """\n""" )

      _a				:							List[Any]							       =      0
      with open(__lowerCAmelCase		,     """w"""		,     encoding="""utf-8""" ) as writer:
          writer.write("""#version: 0.2\n""" )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,     key=lambda __lowerCAmelCase : kv[1] ):
              if index != token_index:
                  logger.warning(
                      f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                      """ Please check that the tokenizer is not corrupted!""" )
                  _a				:							List[Any]							       =      token_index
              writer.write(""" """.join(__lowerCAmelCase ) + """\n""" )
              index += 1

      return vocab_file, merge_file



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None ):







      '''simple docstring'''


      if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _a				:							int							       =      [self.cls_token_id]
      _a				:							str							       =      [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None		,     __lowerCAmelCase = False ):







      '''simple docstring'''


      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__lowerCAmelCase		,     token_ids_a=__lowerCAmelCase		,     already_has_special_tokens=__lowerCAmelCase )

      if token_ids_a is None:
          return [1] + ([0] * len(__lowerCAmelCase )) + [1]
      return [1] + ([0] * len(__lowerCAmelCase )) + [1, 1] + ([0] * len(__lowerCAmelCase )) + [1]



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase = None ):







      '''simple docstring'''


      _a				:							str							       =      [self.sep_token_id]
      _a				:							int							       =      [self.cls_token_id]

      if token_ids_a is None:
          return len(cls + token_ids_a + sep ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase=False		,     **__lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Any							       =      kwargs.pop("""add_prefix_space"""		,     self.add_prefix_space )
      if (is_split_into_words or add_prefix_space) and (len(__lowerCAmelCase ) > 0 and not text[0].isspace()):
          _a				:							List[str]							       =      """ """ + text
      return (text, kwargs)







import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


__lowerCamelCase								=      logging.getLogger(__name__)


@dataclass
class 	snake_case_			:




       """simple docstring"""






       _lowerCamelCase           =					field(
           metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}	)
       _lowerCamelCase           =					field(default=lowercase__			,			metadata={"""help""": """Set this flag to use fast tokenization."""}	)
       # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
       # or just modify its tokenizer_config.json.
       _lowerCamelCase           =					field(
           default=lowercase__			,			metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,			)







def  _snake_case			(							)  ->  Union[str, Any]:






      '''simple docstring'''







      UpperCAmelCase_	:						Union[str, Any]					   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

      if len(sys.argv     ) == 2 and sys.argv[1].endswith(".json"     ):
            # If we pass only one argument to the script and it's the path to a json file,
            # let's parse it to get our arguments.
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Any					   =		parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
      else:
            UpperCAmelCase_   ,   UpperCAmelCase_   ,   UpperCAmelCase_	:						Union[str, Any]					   =		parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir     )
          and os.listdir(training_args.output_dir     )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
                " --overwrite_output_dir to overcome."     )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,	datefmt="%m/%d/%Y %H:%M:%S"       ,	level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,	)
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"       ,	training_args.local_rank       ,	training_args.device       ,	training_args.n_gpu       ,	bool(training_args.local_rank != -1     )       ,	training_args.fpaa       ,	)
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank     ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"       ,	__snake_case     )

      # Prepare Question-Answering task
      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      UpperCAmelCase_	:						Any					   =		AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	)
      UpperCAmelCase_	:						List[str]					   =		AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,	cache_dir=model_args.cache_dir       ,	use_fast=__snake_case       ,	)
      UpperCAmelCase_	:						Optional[int]					   =		AutoModelForQuestionAnswering.from_pretrained(
          model_args.model_name_or_path       ,	from_tf=bool(".ckpt" in model_args.model_name_or_path     )       ,	config=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)

      # Get datasets
      UpperCAmelCase_	:						str					   =		hasattr(model.config       ,	"lang2id"     )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir     )
          if training_args.do_train
          else None
      )
      UpperCAmelCase_	:						Optional[Any]					   =		(
          SquadDataset(
              __snake_case       ,	tokenizer=__snake_case       ,	mode="dev"       ,	is_language_sensitive=__snake_case       ,	cache_dir=model_args.cache_dir       ,	)
          if training_args.do_eval
          else None
      )

      # Data collator
      UpperCAmelCase_	:						Optional[Any]					   =		DataCollatorWithPadding(__snake_case       ,	pad_to_multiple_of=8     ) if training_args.fpaa else None

      # Initialize our Trainer
      UpperCAmelCase_	:						Any					   =		Trainer(
          model=__snake_case       ,	args=__snake_case       ,	train_dataset=__snake_case       ,	eval_dataset=__snake_case       ,	data_collator=__snake_case       ,	)

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir     )







def  _snake_case			(							__snake_case     )  ->  List[str]:






      '''simple docstring'''







      main()


if __name__ == "__main__":
   main()
