
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase			:     str           =   logging.get_logger(__name__)

__lowerCAmelCase			:     Dict           =   '▁'

__lowerCAmelCase			:     Optional[Any]           =   {'vocab_file': 'sentencepiece.bpe.model'}

__lowerCAmelCase			:     Union[str, Any]           =   {
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__lowerCAmelCase			:     Union[str, Any]           =   {
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
__lowerCAmelCase			:     Optional[Any]           =   ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']




class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  a__           =		VOCAB_FILES_NAMES
  a__           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a__           =		PRETRAINED_VOCAB_FILES_MAP
  a__           =		["""input_ids""", """attention_mask"""]

  a__           =		[]
  a__           =		[]
  def __init__( self			: List[str]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: Tuple="<s>"			,							UpperCamelCase__			: int="</s>"			,							UpperCamelCase__			: Any="</s>"			,							UpperCamelCase__			: int="<s>"			,							UpperCamelCase__			: Dict="<unk>"			,							UpperCamelCase__			: Tuple="<pad>"			,							UpperCamelCase__			: str="<mask>"			,							UpperCamelCase__			: Dict=None			,							UpperCamelCase__			: str=None			,							UpperCamelCase__			: Any=None			,							UpperCamelCase__			: Optional[Dict[str, Any]] = None			,							UpperCamelCase__			: Optional[int]=None			,							**UpperCamelCase__			: int			,							)			->	List[Any]:







    """simple docstring"""






    __magic_name__				     =       AddedToken(UpperCamelCase__			,							lstrip=UpperCamelCase__			,							rstrip=UpperCamelCase__    ) if isinstance(UpperCamelCase__			,							UpperCamelCase__    ) else mask_token

    __magic_name__				     =       {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        bos_token=UpperCamelCase__			,							eos_token=UpperCamelCase__			,							unk_token=UpperCamelCase__			,							sep_token=UpperCamelCase__			,							cls_token=UpperCamelCase__			,							pad_token=UpperCamelCase__			,							mask_token=UpperCamelCase__			,							tokenizer_file=UpperCamelCase__			,							src_lang=UpperCamelCase__			,							tgt_lang=UpperCamelCase__			,							additional_special_tokens=UpperCamelCase__			,							sp_model_kwargs=self.sp_model_kwargs			,							**UpperCamelCase__			,							)

    __magic_name__				     =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
    self.sp_model.Load(str(UpperCamelCase__    )    )
    __magic_name__				     =       vocab_file

    # Original fairseq vocab and spm vocab must be "aligned":
    # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
    # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
    # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
    # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

    # Mimic fairseq token-to-id alignment for the first 4 token
    __magic_name__				     =       {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

    # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
    __magic_name__				     =       1

    __magic_name__				     =       len(self.sp_model    )
    __magic_name__				     =       {
        code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase__    )
    }
    __magic_name__				     =       {v: k for k, v in self.lang_code_to_id.items()}
    __magic_name__				     =       len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

    self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
    __magic_name__				     =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
    __magic_name__				     =       list(self.lang_code_to_id.keys()    )

    if additional_special_tokens is not None:
      # Only add those special tokens if they are not already there.
      self._additional_special_tokens.extend(
          [t for t in additional_special_tokens if t not in self._additional_special_tokens]    )

    __magic_name__				     =       src_lang if src_lang is not None else """en_XX"""
    __magic_name__				     =       self.lang_code_to_id[self._src_lang]
    __magic_name__				     =       tgt_lang
    self.set_src_lang_special_tokens(self._src_lang    )
  def __getstate__( self			: int    )			->	int:







    """simple docstring"""






    __magic_name__				     =       self.__dict__.copy()
    __magic_name__				     =       None
    __magic_name__				     =       self.sp_model.serialized_model_proto()
    return state
  def __setstate__( self			: Tuple			,							UpperCamelCase__			: int    )			->	Dict:







    """simple docstring"""






    __magic_name__				     =       d

    # for backward compatibility
    if not hasattr(self			,							"""sp_model_kwargs"""    ):
      __magic_name__				     =       {}

    __magic_name__				     =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )
  @property
  def 							_lowercase							( self			: Dict    )			->	str:







    """simple docstring"""






    return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token
  @property
  def 							_lowercase							( self			: Union[str, Any]    )			->	str:







    """simple docstring"""






    return self._src_lang
  @src_lang.setter
  def 							_lowercase							( self			: Optional[int]			,							UpperCamelCase__			: str    )			->	None:







    """simple docstring"""






    __magic_name__				     =       new_src_lang
    self.set_src_lang_special_tokens(self._src_lang    )
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: List[int]			,							UpperCamelCase__			: Optional[List[int]] = None			,							UpperCamelCase__			: bool = False    )			->	List[int]:







    """simple docstring"""






    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=UpperCamelCase__			,							token_ids_a=UpperCamelCase__			,							already_has_special_tokens=UpperCamelCase__    )

    __magic_name__				     =       [1] * len(self.prefix_tokens    )
    __magic_name__				     =       [1] * len(self.suffix_tokens    )
    if token_ids_a is None:
      return prefix_ones + ([0] * len(UpperCamelCase__    )) + suffix_ones
    return prefix_ones + ([0] * len(UpperCamelCase__    )) + ([0] * len(UpperCamelCase__    )) + suffix_ones
  def 							_lowercase							( self			: List[Any]			,							UpperCamelCase__			: List[int]			,							UpperCamelCase__			: Optional[List[int]] = None    )			->	List[int]:







    """simple docstring"""






    if token_ids_a is None:
      return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: List[int]			,							UpperCamelCase__			: Optional[List[int]] = None    )			->	List[int]:







    """simple docstring"""






    __magic_name__				     =       [self.sep_token_id]
    __magic_name__				     =       [self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep    ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]
  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Optional[str]			,							UpperCamelCase__			: Optional[str]			,							**UpperCamelCase__			: int    )			->	List[Any]:







    """simple docstring"""






    if src_lang is None or tgt_lang is None:
      raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
    __magic_name__				     =       src_lang
    __magic_name__				     =       self(UpperCamelCase__			,							add_special_tokens=UpperCamelCase__			,							return_tensors=UpperCamelCase__			,							**UpperCamelCase__    )
    __magic_name__				     =       self.convert_tokens_to_ids(UpperCamelCase__    )
    __magic_name__				     =       tgt_lang_id
    return inputs
  def 							_lowercase							( self			: Optional[int]    )			->	Any:







    """simple docstring"""






    __magic_name__				     =       {self.convert_ids_to_tokens(UpperCamelCase__    ): i for i in range(self.vocab_size    )}
    vocab.update(self.added_tokens_encoder    )
    return vocab
  def 							_lowercase							( self			: Optional[Any]			,							UpperCamelCase__			: str    )			->	List[str]:







    """simple docstring"""






    return self.sp_model.encode(UpperCamelCase__			,							out_type=UpperCamelCase__    )
  def 							_lowercase							( self			: Dict			,							UpperCamelCase__			: Optional[int]    )			->	List[str]:







    """simple docstring"""






    if token in self.fairseq_tokens_to_ids:
      return self.fairseq_tokens_to_ids[token]
    __magic_name__				     =       self.sp_model.PieceToId(UpperCamelCase__    )

    # Need to return unknown token if the SP model returned 0
    return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: List[Any]    )			->	Any:







    """simple docstring"""






    if index in self.fairseq_ids_to_tokens:
      return self.fairseq_ids_to_tokens[index]
    return self.sp_model.IdToPiece(index - self.fairseq_offset    )
  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Tuple    )			->	str:







    """simple docstring"""






    __magic_name__				     =       """""".join(UpperCamelCase__    ).replace(UpperCamelCase__			,							""" """    ).strip()
    return out_string
  def 							_lowercase							( self			: Dict			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Optional[str] = None    )			->	Tuple[str]:







    """simple docstring"""






    if not os.path.isdir(UpperCamelCase__    ):
      logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
      return
    __magic_name__				     =       os.path.join(
        UpperCamelCase__			,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

    if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCamelCase__    ) and os.path.isfile(self.vocab_file    ):
      copyfile(self.vocab_file			,							UpperCamelCase__    )
    elif not os.path.isfile(self.vocab_file    ):
      with open(UpperCamelCase__			,							"""wb"""    ) as fi:
        __magic_name__				     =       self.sp_model.serialized_model_proto()
        fi.write(UpperCamelCase__    )

    return (out_vocab_file,)
  def 							_lowercase							( self			: List[str]			,							UpperCamelCase__			: List[str]			,							UpperCamelCase__			: str = "en_XX"			,							UpperCamelCase__			: Optional[List[str]] = None			,							UpperCamelCase__			: str = "ro_RO"			,							**UpperCamelCase__			: int			,							)			->	BatchEncoding:







    """simple docstring"""






    __magic_name__				     =       src_lang
    __magic_name__				     =       tgt_lang
    return super().prepare_seqaseq_batch(UpperCamelCase__			,							UpperCamelCase__			,							**UpperCamelCase__    )
  def 							_lowercase							( self			: Optional[int]    )			->	List[str]:







    """simple docstring"""






    return self.set_src_lang_special_tokens(self.src_lang    )
  def 							_lowercase							( self			: Optional[Any]    )			->	Union[str, Any]:







    """simple docstring"""






    return self.set_tgt_lang_special_tokens(self.tgt_lang    )
  def 							_lowercase							( self			: Union[str, Any]			,							UpperCamelCase__			: str    )			->	None:







    """simple docstring"""






    __magic_name__				     =       self.lang_code_to_id[src_lang]
    __magic_name__				     =       []
    __magic_name__				     =       [self.eos_token_id, self.cur_lang_code]





  def 							_lowercase							( self			: Tuple			,							UpperCamelCase__			: str    )			->	None:







    """simple docstring"""






    __magic_name__				     =       self.lang_code_to_id[lang]
    __magic_name__				     =       []
    __magic_name__				     =       [self.eos_token_id, self.cur_lang_code]









from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__				   =		logging.get_logger(__name__)

UpperCAmelCase__				   =		{}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  '''llama'''
      __snake_case       =  ['''past_key_values''']







      def __init__(      self      :						Optional[Any] ,					__UpperCAmelCase      :						Union[str, Any]=32_000 ,					__UpperCAmelCase      :						str=4_096 ,					__UpperCAmelCase      :						int=11_008 ,					__UpperCAmelCase      :						Tuple=32 ,					__UpperCAmelCase      :						Optional[int]=32 ,					__UpperCAmelCase      :						List[str]=None ,					__UpperCAmelCase      :						Union[str, Any]="silu" ,					__UpperCAmelCase      :						Tuple=2_048 ,					__UpperCAmelCase      :						Optional[Any]=0.02 ,					__UpperCAmelCase      :						Any=1e-6 ,					__UpperCAmelCase      :						Union[str, Any]=True ,					__UpperCAmelCase      :						Optional[int]=0 ,					__UpperCAmelCase      :						Optional[int]=1 ,					__UpperCAmelCase      :						Optional[int]=2 ,					__UpperCAmelCase      :						Tuple=1 ,					__UpperCAmelCase      :						List[str]=False ,					__UpperCAmelCase      :						Tuple=None ,					**__UpperCAmelCase      :						Tuple ,					)    ->str:




       """simple docstring"""


       a		  =      vocab_size
       a		  =      max_position_embeddings
       a		  =      hidden_size
       a		  =      intermediate_size
       a		  =      num_hidden_layers
       a		  =      num_attention_heads

       # for backward compatibility
       if num_key_value_heads is None:
        a		  =      num_attention_heads

       a		  =      num_key_value_heads
       a		  =      hidden_act
       a		  =      initializer_range
       a		  =      rms_norm_eps
       a		  =      pretraining_tp
       a		  =      use_cache
       a		  =      rope_scaling
       self._rope_scaling_validation()

       super().__init__(
           pad_token_id=__UpperCAmelCase ,					bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					tie_word_embeddings=__UpperCAmelCase ,					**__UpperCAmelCase ,					)







      def       __lowerCAmelCase					(      self      :						Tuple  )    ->Tuple:




       """simple docstring"""


       if self.rope_scaling is None:
        return

       if not isinstance(self.rope_scaling ,					__UpperCAmelCase  ) or len(self.rope_scaling  ) != 2:
        raise ValueError(
            '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
            F"""got {self.rope_scaling}"""  )
       a		  =      self.rope_scaling.get('''type''' ,					__UpperCAmelCase  )
       a		  =      self.rope_scaling.get('''factor''' ,					__UpperCAmelCase  )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
        raise ValueError(
            F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""  )
       if rope_scaling_factor is None or not isinstance(__UpperCAmelCase ,					__UpperCAmelCase  ) or rope_scaling_factor <= 1.0:
        raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""  )


