






"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_mbartaa import MBartaaTokenizer
else:
    _UpperCamelCase						:    Tuple		=						None


_UpperCamelCase						:    str		=						logging.get_logger(__name__)

_UpperCamelCase						:    Tuple		=						{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase						:    str		=						{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

_UpperCamelCase						:    Optional[int]		=						{
    'facebook/mbart-large-50-one-to-many-mmt': 1_0_2_4,
}

# fmt: off
_UpperCamelCase						:    List[Any]		=						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']

class 			a	(  a_      ):
    UpperCAmelCase_					:	Any      	=VOCAB_FILES_NAMES
    UpperCAmelCase_					:	str      	=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_					:	Union[str, Any]      	=PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_					:	Optional[int]      	=["input_ids", "attention_mask"]
    UpperCAmelCase_					:	Optional[Any]      	=MBartaaTokenizer

    UpperCAmelCase_					:	List[int]      	=[]
    UpperCAmelCase_					:	List[int]      	=[]







    def __init__(     self				,						_lowerCamelCase=None				,						_lowerCamelCase=None				,						_lowerCamelCase=None				,						_lowerCamelCase=None				,						_lowerCamelCase="</s>"				,						_lowerCamelCase="</s>"				,						_lowerCamelCase="<s>"				,						_lowerCamelCase="<unk>"				,						_lowerCamelCase="<pad>"				,						_lowerCamelCase="<mask>"				,						**_lowerCamelCase				,						):
      # Mask token behave like a normal word, i.e. include the space before it
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else mask_token

      lowercase				       =			kwargs.get('additional_special_tokens'				,						[]			)
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          _lowerCamelCase				,						src_lang=_lowerCamelCase				,						tgt_lang=_lowerCamelCase				,						tokenizer_file=_lowerCamelCase				,						eos_token=_lowerCamelCase				,						sep_token=_lowerCamelCase				,						cls_token=_lowerCamelCase				,						unk_token=_lowerCamelCase				,						pad_token=_lowerCamelCase				,						mask_token=_lowerCamelCase				,						**_lowerCamelCase				,						)

      lowercase				       =			vocab_file
      lowercase				       =			False if not self.vocab_file else True

      lowercase				       =			{
          lang_code: self.convert_tokens_to_ids(_lowerCamelCase			) for lang_code in FAIRSEQ_LANGUAGE_CODES
      }

      lowercase				       =			src_lang if src_lang is not None else 'en_XX'
      lowercase				       =			tgt_lang
      lowercase				       =			self.lang_code_to_id[self._src_lang]
      self.set_src_lang_special_tokens(self._src_lang			)







    @property
    def      UpperCamelCase_ (     self			):
      return self._src_lang







    @src_lang.setter
    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			new_src_lang
      self.set_src_lang_special_tokens(self._src_lang			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = "en_XX"				,						_lowerCamelCase = None				,						_lowerCamelCase = "ro_RO"				,						**_lowerCamelCase				,						):
      lowercase				       =			src_lang
      lowercase				       =			tgt_lang
      return super().prepare_seqaseq_batch(_lowerCamelCase				,						_lowerCamelCase				,						**_lowerCamelCase			)







    def      UpperCamelCase_ (     self			):
      return self.set_src_lang_special_tokens(self.src_lang			)







    def      UpperCamelCase_ (     self			):
      return self.set_tgt_lang_special_tokens(self.tgt_lang			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			self.convert_tokens_to_ids(_lowerCamelCase			)
      lowercase				       =			[self.cur_lang_code_id]
      lowercase				       =			[self.eos_token_id]

      lowercase				       =			self.convert_ids_to_tokens(self.prefix_tokens			)
      lowercase				       =			self.convert_ids_to_tokens(self.suffix_tokens			)

      lowercase				       =			processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,						pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,						self.prefix_tokens + self.suffix_tokens			)			)				,						)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			self.convert_tokens_to_ids(_lowerCamelCase			)
      lowercase				       =			[self.cur_lang_code_id]
      lowercase				       =			[self.eos_token_id]

      lowercase				       =			self.convert_ids_to_tokens(self.prefix_tokens			)
      lowercase				       =			self.convert_ids_to_tokens(self.suffix_tokens			)

      lowercase				       =			processors.TemplateProcessing(
          single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,						pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,						special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,						self.prefix_tokens + self.suffix_tokens			)			)				,						)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase				,						**_lowerCamelCase			):
      if src_lang is None or tgt_lang is None:
        raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'			)
      lowercase				       =			src_lang
      lowercase				       =			self(_lowerCamelCase				,						add_special_tokens=_lowerCamelCase				,						return_tensors=_lowerCamelCase				,						**_lowerCamelCase			)
      lowercase				       =			self.convert_tokens_to_ids(_lowerCamelCase			)
      lowercase				       =			tgt_lang_id
      return inputs







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      if not self.can_save_slow_tokenizer:
        raise ValueError(
            'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
            'tokenizer.'			)

      if not os.path.isdir(_lowerCamelCase			):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
        return
      lowercase				       =			os.path.join(
          _lowerCamelCase				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

      if os.path.abspath(self.vocab_file			) != os.path.abspath(_lowerCamelCase			):
        copyfile(self.vocab_file				,						_lowerCamelCase			)

      return (out_vocab_file,)










"""simple docstring"""



import subprocess
from argparse import ArgumentParser
from typing import List, Union

from huggingface_hub.hf_api import HfFolder, create_repo, whoami
from requests.exceptions import HTTPError

from . import BaseTransformersCLICommand

class 			a	(  a_      ):







    @staticmethod
    def      UpperCamelCase_ (     _lowerCamelCase			):
      lowercase				       =			parser.add_parser('login'				,						help='Log in using the same credentials as on huggingface.co'			)
      login_parser.set_defaults(func=lambda _lowerCamelCase			: LoginCommand(_lowerCamelCase			)			)
      lowercase				       =			parser.add_parser('whoami'				,						help='Find out which huggingface.co account you are logged in as.'			)
      whoami_parser.set_defaults(func=lambda _lowerCamelCase			: WhoamiCommand(_lowerCamelCase			)			)
      lowercase				       =			parser.add_parser('logout'				,						help='Log out'			)
      logout_parser.set_defaults(func=lambda _lowerCamelCase			: LogoutCommand(_lowerCamelCase			)			)

      # new system: git-based repo system
      lowercase				       =			parser.add_parser(
          'repo'				,						help='Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.'				,						)
      lowercase				       =			repo_parser.add_subparsers(
          help='Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands'			)
      lowercase				       =			repo_subparsers.add_parser(
          'create'				,						help='Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co'			)
      repo_create_parser.add_argument(
          'name'				,						type=_lowerCamelCase				,						help='Name for your model\'s repo. Will be namespaced under your username to build the model id.'				,						)
      repo_create_parser.add_argument('--organization'				,						type=_lowerCamelCase				,						help='Optional: organization namespace.'			)
      repo_create_parser.add_argument('-y'				,						'--yes'				,						action='store_true'				,						help='Optional: answer Yes to the prompt'			)
      repo_create_parser.set_defaults(func=lambda _lowerCamelCase			: RepoCreateCommand(_lowerCamelCase			)			)


class 			a	:
    UpperCAmelCase_					:	Union[str, Any]      	="\u001b[1m"
    UpperCAmelCase_					:	Optional[Any]      	="\u001b[31m"
    UpperCAmelCase_					:	List[str]      	="\u001b[90m"
    UpperCAmelCase_					:	str      	="\u001b[0m"







    @classmethod
    def      UpperCamelCase_ (     cls				,						_lowerCamelCase			):
      return F'{cls._bold}{s}{cls._reset}'







    @classmethod
    def      UpperCamelCase_ (     cls				,						_lowerCamelCase			):
      return F'{cls._bold}{cls._red}{s}{cls._reset}'







    @classmethod
    def      UpperCamelCase_ (     cls				,						_lowerCamelCase			):
      return F'{cls._gray}{s}{cls._reset}'


def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    List[List[Union[str, int]]]						,   __snake_case     :    List[str]       ):





  '''simple docstring'''





  lowercase				       =			[max(len(str(__snake_case       )       ) for x in col       ) for col in zip(*__snake_case						,   __snake_case       )]
  lowercase				       =			('{{:{}}} ' * len(__snake_case       )).format(*__snake_case       )
  lowercase				       =			[]
  lines.append(row_format.format(*__snake_case       )       )
  lines.append(row_format.format(*['-' * w for w in col_widths]       )       )
  for row in rows:
    lines.append(row_format.format(*__snake_case       )       )
  return "\n".join(__snake_case       )

class 			a	:







    def __init__(     self				,						_lowerCamelCase			):
      lowercase				       =			args


class 			a	(  a_      ):







    def      UpperCamelCase_ (     self			):
      print(
          ANSI.red(
              'ERROR! `huggingface-cli login` uses an outdated login mechanism '
              'that is not compatible with the Hugging Face Hub backend anymore. '
              'Please use `huggingface-cli login instead.'			)			)


class 			a	(  a_      ):







    def      UpperCamelCase_ (     self			):
      print(
          ANSI.red(
              'WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use '
              '`huggingface-cli whoami` instead.'			)			)
      lowercase				       =			HfFolder.get_token()
      if token is None:
        print('Not logged in'			)
        exit()
      try:
        lowercase       ,	lowercase				       =			whoami(_lowerCamelCase			)
        print(_lowerCamelCase			)
        if orgs:
          print(ANSI.bold('orgs: '			)				,						','.join(_lowerCamelCase			)			)
      except HTTPError as e:
        print(_lowerCamelCase			)
        print(ANSI.red(e.response.text			)			)
        exit(1			)


class 			a	(  a_      ):







    def      UpperCamelCase_ (     self			):
      print(
          ANSI.red(
              'ERROR! `transformers-cli logout` uses an outdated logout mechanism '
              'that is not compatible with the Hugging Face Hub backend anymore. '
              'Please use `huggingface-cli logout instead.'			)			)


class 			a	(  a_      ):







    def      UpperCamelCase_ (     self			):
      print(
          ANSI.red(
              'WARNING! Managing repositories through transformers-cli is deprecated. '
              'Please use `huggingface-cli` instead.'			)			)
      lowercase				       =			HfFolder.get_token()
      if token is None:
        print('Not logged in'			)
        exit(1			)
      try:
        lowercase				       =			subprocess.check_output(['git', '--version']			).decode('utf-8'			)
        print(ANSI.gray(stdout.strip()			)			)
      except FileNotFoundError:
        print('Looks like you do not have git installed, please install.'			)

      try:
        lowercase				       =			subprocess.check_output(['git-lfs', '--version']			).decode('utf-8'			)
        print(ANSI.gray(stdout.strip()			)			)
      except FileNotFoundError:
        print(
            ANSI.red(
                'Looks like you do not have git-lfs installed, please install.'
                ' You can install from https://git-lfs.github.com/.'
                ' Then run `git lfs install` (you only have to do this once).'			)			)
      print(''			)

      lowercase       ,	lowercase				       =			whoami(_lowerCamelCase			)
      lowercase				       =			self.args.organization if self.args.organization is not None else user
      lowercase				       =			F'{namespace}/{self.args.name}'
      print(F'You are about to create {ANSI.bold(_lowerCamelCase			)}'			)

      if not self.args.yes:
        lowercase				       =			input('Proceed? [Y/n] '			).lower()
        if not (choice == "" or choice == "y" or choice == "yes"):
          print('Abort'			)
          exit()
      try:
        lowercase				       =			create_repo(_lowerCamelCase				,						name=self.args.name				,						organization=self.args.organization			)
      except HTTPError as e:
        print(_lowerCamelCase			)
        print(ANSI.red(e.response.text			)			)
        exit(1			)
      print('\nYour repo now lives at:'			)
      print(F'  {ANSI.bold(_lowerCamelCase			)}'			)
      print('\nYou can clone it locally with the command below, and commit/push as usual.'			)
      print(F'\n  git clone {url}'			)
      print(''			)


