




"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a            =							logging.get_logger(__name__)

a            =							'▁'

a            =							{'vocab_file': 'sentencepiece.bpe.model'}

a            =							{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

a            =							{
    'facebook/mbart-large-50-one-to-many-mmt': 1_0_2_4,
}

# fmt: off
a            =							['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class 					SCREAMING_SNAKE_CASE__    (    _snake_case    ):
						_a			  =  VOCAB_FILES_NAMES
						_a			  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a			  =  PRETRAINED_VOCAB_FILES_MAP
						_a			  =  ['''input_ids''', '''attention_mask''']

						_a			  =  []
						_a			  =  []


						def __init__(							self :	Any  ,  lowerCAmelCase :	str  ,  lowerCAmelCase :	List[str]=None  ,  lowerCAmelCase :	List[Any]=None  ,  lowerCAmelCase :	Union[str, Any]="</s>"  ,  lowerCAmelCase :	List[Any]="</s>"  ,  lowerCAmelCase :	Tuple="<s>"  ,  lowerCAmelCase :	Any="<unk>"  ,  lowerCAmelCase :	List[str]="<pad>"  ,  lowerCAmelCase :	List[Any]="<mask>"  ,  lowerCAmelCase :	int = None  ,  **lowerCAmelCase :	str  ,  ):
										# Mask token behave like a normal word, i.e. include the space before it
										lowerCAmelCase              = AddedToken(_UpperCamelCase  ,  lstrip=_UpperCamelCase  ,  rstrip=_UpperCamelCase    ) if isinstance(_UpperCamelCase  ,  _UpperCamelCase    ) else mask_token

										lowerCAmelCase              = {} if sp_model_kwargs is None else sp_model_kwargs

										lowerCAmelCase              = kwargs.get("""additional_special_tokens"""  ,  []    )
										kwargs["additional_special_tokens"] += [
										    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
										]

										super().__init__(
										    src_lang=_UpperCamelCase  ,  tgt_lang=_UpperCamelCase  ,  eos_token=_UpperCamelCase  ,  unk_token=_UpperCamelCase  ,  sep_token=_UpperCamelCase  ,  cls_token=_UpperCamelCase  ,  pad_token=_UpperCamelCase  ,  mask_token=_UpperCamelCase  ,  sp_model_kwargs=self.sp_model_kwargs  ,  **_UpperCamelCase  ,  )

										lowerCAmelCase              = spm.SentencePieceProcessor(**self.sp_model_kwargs    )
										self.sp_model.Load(str(_UpperCamelCase    )    )
										lowerCAmelCase              = vocab_file

										# Original fairseq vocab and spm vocab must be "aligned":
										# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
										# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
										# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
										# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

										# Mimic fairseq token-to-id alignment for the first 4 token
										lowerCAmelCase              = {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

										# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
										lowerCAmelCase              = 1

										lowerCAmelCase              = len(self.sp_model    )
										lowerCAmelCase              = {
										    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCamelCase    )
										}
										lowerCAmelCase              = {v: k for k, v in self.lang_code_to_id.items()}
										lowerCAmelCase              = len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

										self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
										lowerCAmelCase              = {v: k for k, v in self.fairseq_tokens_to_ids.items()}

										lowerCAmelCase              = src_lang if src_lang is not None else 'en_XX'
										lowerCAmelCase              = self.lang_code_to_id[self._src_lang]
										lowerCAmelCase              = tgt_lang
										self.set_src_lang_special_tokens(self._src_lang    )


						@property
						def      __lowercase       (							self :	str    ):
										return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token


						@property
						def      __lowercase       (							self :	Optional[Any]    ):
										return self._src_lang


						@src_lang.setter
						def      __lowercase       (							self :	List[Any]  ,  lowerCAmelCase :	Tuple    ):
										lowerCAmelCase              = new_src_lang
										self.set_src_lang_special_tokens(self._src_lang    )


						def __getstate__(							self :	List[str]    ):
										lowerCAmelCase              = self.__dict__.copy()
										lowerCAmelCase              = None
										return state


						def __setstate__(							self :	Optional[Any]  ,  lowerCAmelCase :	Tuple    ):
										lowerCAmelCase              = d

										# for backward compatibility
										if not hasattr(self  ,  """sp_model_kwargs"""    ):
														lowerCAmelCase              = {}

										lowerCAmelCase              = spm.SentencePieceProcessor(**self.sp_model_kwargs    )
										self.sp_model.Load(self.vocab_file    )


						def      __lowercase       (							self :	Optional[Any]    ):
										lowerCAmelCase              = {self.convert_ids_to_tokens(_UpperCamelCase    ): i for i in range(self.vocab_size    )}
										vocab.update(self.added_tokens_encoder    )
										return vocab


						def      __lowercase       (							self :	List[str]  ,  lowerCAmelCase :	Dict    ):
										return self.sp_model.encode(_UpperCamelCase  ,  out_type=_UpperCamelCase    )


						def      __lowercase       (							self :	int  ,  lowerCAmelCase :	List[Any]    ):
										if token in self.fairseq_tokens_to_ids:
														return self.fairseq_tokens_to_ids[token]
										lowerCAmelCase              = self.sp_model.PieceToId(_UpperCamelCase    )

										# Need to return unknown token if the SP model returned 0
										return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


						def      __lowercase       (							self :	Any  ,  lowerCAmelCase :	int    ):
										if index in self.fairseq_ids_to_tokens:
														return self.fairseq_ids_to_tokens[index]
										return self.sp_model.IdToPiece(index - self.fairseq_offset    )


						def      __lowercase       (							self :	Optional[Any]  ,  lowerCAmelCase :	int    ):
										lowerCAmelCase              = []
										lowerCAmelCase              = ''
										lowerCAmelCase              = False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		if not prev_is_special:
																						out_string += " "
																		out_string += self.sp_model.decode(_UpperCamelCase    ) + token
																		lowerCAmelCase              = True
																		lowerCAmelCase              = []
														else:
																		current_sub_tokens.append(_UpperCamelCase    )
																		lowerCAmelCase              = False
										out_string += self.sp_model.decode(_UpperCamelCase    )
										return out_string.strip()


						def      __lowercase       (							self :	Any  ,  lowerCAmelCase :	Optional[Any]  ,  lowerCAmelCase :	Optional[Any] = None    ):
										if not os.path.isdir(_UpperCamelCase    ):
														logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
														return
										lowerCAmelCase              = os.path.join(
										    _UpperCamelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

										if os.path.abspath(self.vocab_file    ) != os.path.abspath(_UpperCamelCase    ) and os.path.isfile(self.vocab_file    ):
														copyfile(self.vocab_file  ,  _UpperCamelCase    )
										elif not os.path.isfile(self.vocab_file    ):
														with open(_UpperCamelCase  ,  """wb"""    ) as fi:
																		lowerCAmelCase              = self.sp_model.serialized_model_proto()
																		fi.write(_UpperCamelCase    )

										return (out_vocab_file,)


						def      __lowercase       (							self :	int  ,  lowerCAmelCase :	Union[str, Any]  ,  lowerCAmelCase :	Optional[Any] = None  ,  lowerCAmelCase :	Optional[Any] = False    ):

										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_UpperCamelCase  ,  token_ids_a=_UpperCamelCase  ,  already_has_special_tokens=_UpperCamelCase    )

										lowerCAmelCase              = [1] * len(self.prefix_tokens    )
										lowerCAmelCase              = [1] * len(self.suffix_tokens    )
										if token_ids_a is None:
														return prefix_ones + ([0] * len(_UpperCamelCase    )) + suffix_ones
										return prefix_ones + ([0] * len(_UpperCamelCase    )) + ([0] * len(_UpperCamelCase    )) + suffix_ones


						def      __lowercase       (							self :	Tuple  ,  lowerCAmelCase :	Optional[Any]  ,  lowerCAmelCase :	Dict = None    ):
										if token_ids_a is None:
														return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


						def      __lowercase       (							self :	List[Any]  ,  lowerCAmelCase :	Union[str, Any]  ,  lowerCAmelCase :	List[Any]  ,  lowerCAmelCase :	Union[str, Any]  ,  lowerCAmelCase :	List[Any]  ,  **lowerCAmelCase :	Optional[int]    ):
										if src_lang is None or tgt_lang is None:
														raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
										lowerCAmelCase              = src_lang
										lowerCAmelCase              = self(_UpperCamelCase  ,  add_special_tokens=_UpperCamelCase  ,  return_tensors=_UpperCamelCase  ,  **_UpperCamelCase    )
										lowerCAmelCase              = self.convert_tokens_to_ids(_UpperCamelCase    )
										lowerCAmelCase              = tgt_lang_id
										return inputs


						def      __lowercase       (							self :	str  ,  lowerCAmelCase :	Tuple  ,  lowerCAmelCase :	Tuple = "en_XX"  ,  lowerCAmelCase :	Optional[int] = None  ,  lowerCAmelCase :	Optional[int] = "ro_RO"  ,  **lowerCAmelCase :	Any  ,  ):
										lowerCAmelCase              = src_lang
										lowerCAmelCase              = tgt_lang
										return super().prepare_seqaseq_batch(_UpperCamelCase  ,  _UpperCamelCase  ,  **_UpperCamelCase    )


						def      __lowercase       (							self :	str    ):
										return self.set_src_lang_special_tokens(self.src_lang    )


						def      __lowercase       (							self :	Optional[int]    ):
										return self.set_tgt_lang_special_tokens(self.tgt_lang    )


						def      __lowercase       (							self :	Tuple  ,  lowerCAmelCase :	int    ):
										lowerCAmelCase              = self.lang_code_to_id[src_lang]
										lowerCAmelCase              = [self.cur_lang_code_id]
										lowerCAmelCase              = [self.eos_token_id]




						def      __lowercase       (							self :	str  ,  lowerCAmelCase :	str    ):
										lowerCAmelCase              = self.lang_code_to_id[tgt_lang]
										lowerCAmelCase              = [self.cur_lang_code_id]
										lowerCAmelCase              = [self.eos_token_id]













def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if not head:
										return True
					# split the list to two parts
					UpperCAmelCase_       ,  UpperCAmelCase_    :      Any            =						head.next, head
					while fast and fast.next:
										UpperCAmelCase_    :      str            =						fast.next.next
										UpperCAmelCase_    :      Union[str, Any]            =						slow.next
					UpperCAmelCase_    :      int            =						slow.next
					UpperCAmelCase_    :      List[Any]            =						None  # Don't forget here! But forget still works!
					# reverse the second part
					UpperCAmelCase_    :      Tuple            =						None
					while second:
										UpperCAmelCase_    :      int            =						second.next
										UpperCAmelCase_    :      Any            =						node
										UpperCAmelCase_    :      Optional[Any]            =						second
										UpperCAmelCase_    :      Tuple            =						nxt
					# compare two parts
					# second part has the same or one less node
					while node:
										if node.val != head.val:
															return False
										UpperCAmelCase_    :      Optional[Any]            =						node.next
										UpperCAmelCase_    :      Dict            =						head.next
					return True




def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					if not head or not head.next:
										return True

					# 1. Get the midpoint (slow)
					UpperCAmelCase_    :      Any            =						head
					while fast and fast.next:
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Tuple            =						fast.next.next, slow.next

					# 2. Push the second half into the stack
					UpperCAmelCase_    :      List[str]            =						[slow.val]
					while slow.next:
										UpperCAmelCase_    :      List[str]            =						slow.next
										stack.append(slow.val  )

					# 3. Comparison
					while stack:
										if stack.pop() != cur.val:
															return False
										UpperCAmelCase_    :      int            =						cur.next

					return True






def      lowercase__      (     __snake_case    :							Dict  ):


					'''simple docstring'''


					if not head or not head.next:
										return True
					UpperCAmelCase_    :      Tuple            =						{}
					UpperCAmelCase_    :      int            =						0
					while head:
										if head.val in d:
															d[head.val].append(__snake_case  )
										else:
															UpperCAmelCase_    :      List[Any]            =						[pos]
										UpperCAmelCase_    :      Any            =						head.next
										pos += 1
					UpperCAmelCase_    :      Dict            =						pos - 1
					UpperCAmelCase_    :      Optional[int]            =						0
					for v in d.values():
										if len(__snake_case  ) % 2 != 0:
															middle += 1
										else:
															UpperCAmelCase_    :      int            =						0
															for i in range(0     ,       len(__snake_case  )  ):
																				if v[i] + v[len(__snake_case  ) - 1 - step] != checksum:
																									return False
																				step += 1
										if middle > 1:
															return False
					return True






