



'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase      						=	logging.get_logger(__name__)

_lowerCAmelCase      						=	{'''vocab_file''': '''spm_char.model'''}

_lowerCAmelCase      						=	{
    '''vocab_file''': {
        '''microsoft/speecht5_asr''': '''https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model''',
        '''microsoft/speecht5_tts''': '''https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model''',
        '''microsoft/speecht5_vc''': '''https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model''',
    }
}

_lowerCAmelCase      						=	{
    '''microsoft/speecht5_asr''': 1024,
    '''microsoft/speecht5_tts''': 1024,
    '''microsoft/speecht5_vc''': 1024,
}






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      Dict				=      VOCAB_FILES_NAMES
  __lowercase							:      Optional[Any]				=      PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:      Optional[int]				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowercase							:      str				=      ['''input_ids''', '''attention_mask''']




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase="<s>"  ,__UpperCAmelCase="</s>"  ,__UpperCAmelCase="<unk>"  ,__UpperCAmelCase="<pad>"  ,__UpperCAmelCase = None  ,**__UpperCAmelCase  ,)    ->		None:
      lowerCAmelCase__ :  Any							      =					{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=__UpperCAmelCase  ,eos_token=__UpperCAmelCase  ,unk_token=__UpperCAmelCase  ,pad_token=__UpperCAmelCase  ,sp_model_kwargs=self.sp_model_kwargs  ,**__UpperCAmelCase  ,)

      lowerCAmelCase__ :  Dict							      =					vocab_file

      lowerCAmelCase__ :  int							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(__UpperCAmelCase      )




  @property
  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      return self.sp_model.get_piece_size()




  def 							UpperCAmelCase_	(    self      )    ->		Optional[int]:
      lowerCAmelCase__ :  List[str]							      =					{self.convert_ids_to_tokens(__UpperCAmelCase      ): i for i in range(self.vocab_size      )}
      vocab.update(self.added_tokens_encoder      )
      return vocab




  def __getstate__(    self      )    ->		List[str]:
      lowerCAmelCase__ :  Any							      =					self.__dict__.copy()
      lowerCAmelCase__ :  Dict							      =					None
      return state




  def __setstate__(    self  ,__UpperCAmelCase      )    ->		List[str]:
      lowerCAmelCase__ :  List[str]							      =					d

      # for backward compatibility
      if not hasattr(self  ,"""sp_model_kwargs"""      ):
          lowerCAmelCase__ :  int							      =					{}

      lowerCAmelCase__ :  Optional[int]							      =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.vocab_file      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		List[str]:
      return self.sp_model.encode(__UpperCAmelCase  ,out_type=__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		str:
      return self.sp_model.piece_to_id(__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		str:
      lowerCAmelCase__ :  Tuple							      =					self.sp_model.IdToPiece(__UpperCAmelCase      )
      return token




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		str:
      lowerCAmelCase__ :  Union[str, Any]							      =					[]
      lowerCAmelCase__ :  Optional[Any]							      =					""""""
      for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
              out_string += self.sp_model.decode(__UpperCAmelCase      ) + token
              lowerCAmelCase__ :  Dict							      =					[]
          else:
              current_sub_tokens.append(__UpperCAmelCase      )
      out_string += self.sp_model.decode(__UpperCAmelCase      )
      return out_string.strip()




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase=None      )    ->		List[int]:
      if token_ids_a is None:
          return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None  ,__UpperCAmelCase = False      )    ->		List[int]:
      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__UpperCAmelCase  ,token_ids_a=__UpperCAmelCase  ,already_has_special_tokens=__UpperCAmelCase      )

      lowerCAmelCase__ :  str							      =					[1]
      if token_ids_a is None:
          return ([0] * len(__UpperCAmelCase      )) + suffix_ones
      return ([0] * len(__UpperCAmelCase      )) + ([0] * len(__UpperCAmelCase      )) + suffix_ones






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      lowerCAmelCase__ :  Dict							      =					os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file  ,__UpperCAmelCase      )
      elif not os.path.isfile(self.vocab_file      ):
          with open(__UpperCAmelCase  ,"""wb"""      ) as fi:
              lowerCAmelCase__ :  str							      =					self.sp_model.serialized_model_proto()
              fi.write(__UpperCAmelCase      )

      return (out_vocab_file,)







'''simple docstring'''





# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


_lowerCAmelCase      						=	'''3'''

print('''Python version:''', sys.version)
print('''transformers version:''', transformers.__version__)

try:
 import torch

 print('''Torch version:''', torch.__version__)
 print('''Cuda available:''', torch.cuda.is_available())
 print('''Cuda version:''', torch.version.cuda)
 print('''CuDNN version:''', torch.backends.cudnn.version())
 print('''Number of GPUs available:''', torch.cuda.device_count())
 print('''NCCL version:''', torch.cuda.nccl.version())
except ImportError:
 print('''Torch version:''', None)

try:
 import deepspeed

 print('''DeepSpeed version:''', deepspeed.__version__)
except ImportError:
 print('''DeepSpeed version:''', None)

try:
 import tensorflow as tf

 print('''TensorFlow version:''', tf.__version__)
 print('''TF GPUs available:''', bool(tf.config.list_physical_devices('''GPU''')))
 print('''Number of TF GPUs available:''', len(tf.config.list_physical_devices('''GPU''')))
except ImportError:
 print('''TensorFlow version:''', None)


