


'''simple docstring'''

from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


a_  :				int						= logging.get_logger(__name__)





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  ["input_features", "attention_mask"]






  def __init__(    self			,  UpperCamelCase=80			,  UpperCamelCase=1_6000			,  UpperCamelCase=80			,  UpperCamelCase=0.0			,  UpperCamelCase=True			,  UpperCamelCase=True			,  UpperCamelCase=True			,  **UpperCamelCase			,  ):







   """simple docstring"""
   super().__init__(feature_size=UpperCamelCase			,  sampling_rate=UpperCamelCase			,  padding_value=UpperCamelCase			,  **UpperCamelCase     )
   lowerCamelCase_      =  num_mel_bins
   lowerCamelCase_      =  do_ceptral_normalize
   lowerCamelCase_      =  normalize_means
   lowerCamelCase_      =  normalize_vars
   lowerCamelCase_      =  True






  def   snake_case		(    self			,  UpperCamelCase			,  ):







   """simple docstring"""
   lowerCamelCase_      =  waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
   lowerCamelCase_      =  torch.from_numpy(UpperCamelCase     ).unsqueeze(0     )
   lowerCamelCase_      =  ta_kaldi.fbank(UpperCamelCase			,  num_mel_bins=self.num_mel_bins			,  sample_frequency=self.sampling_rate     )
   return features.numpy()






  @staticmethod
  def   snake_case		(    UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase = True			,  UpperCamelCase = True			,  UpperCamelCase = 0.0			,  ):







   """simple docstring"""
   # make sure we normalize float32 arrays
   if normalize_means:
    lowerCamelCase_      =  x[:input_length].mean(axis=0     )
    lowerCamelCase_      =  np.subtract(UpperCamelCase			,  UpperCamelCase     )
   if normalize_vars:
    lowerCamelCase_      =  x[:input_length].std(axis=0     )
    lowerCamelCase_      =  np.divide(UpperCamelCase			,  UpperCamelCase     )

   if input_length < x.shape[0]:
    lowerCamelCase_      =  padding_value

   # make sure array is in float32
   lowerCamelCase_      =  x.astype(np.floataa     )

   return x






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  attention_mask.sum(-1     ) if attention_mask is not None else [x.shape[0] for x in input_features]
   return [
       self.utterance_cmvn(UpperCamelCase			,  UpperCamelCase			,  self.normalize_means			,  self.normalize_vars			,  self.padding_value     )
       for x, n in zip(UpperCamelCase			,  UpperCamelCase     )
   ]







  def __call__(    self			,  UpperCamelCase			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""

   if sampling_rate is not None:
    if sampling_rate != self.sampling_rate:
     raise ValueError(
         f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
         f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
         f''' {self.sampling_rate} and not {sampling_rate}.'''     )
   else:
    logger.warning(
        "It is strongly recommended to pass the `sampling_rate` argument to this function. "
        "Failing to do so can result in silent errors that might be hard to debug."     )

   lowerCamelCase_      =  isinstance(UpperCamelCase			,  np.ndarray     ) and len(raw_speech.shape     ) > 1
   if is_batched_numpy and len(raw_speech.shape     ) > 2:
    raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''     )
   lowerCamelCase_      =  is_batched_numpy or (
       isinstance(UpperCamelCase			,  (list, tuple)     ) and (isinstance(raw_speech[0]			,  (np.ndarray, tuple, list)     ))
   )

   if is_batched:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for speech in raw_speech]
   elif not is_batched and not isinstance(UpperCamelCase			,  np.ndarray     ):
    lowerCamelCase_      =  np.asarray(UpperCamelCase			,  dtype=np.floataa     )
   elif isinstance(UpperCamelCase			,  np.ndarray     ) and raw_speech.dtype is np.dtype(np.floataa     ):
    lowerCamelCase_      =  raw_speech.astype(np.floataa     )

   # always return batch
   if not is_batched:
    lowerCamelCase_      =  [raw_speech]

   # extract fbank features
   lowerCamelCase_      =  [self._extract_fbank_features(UpperCamelCase     ) for waveform in raw_speech]

   # convert into correct format for padding
   lowerCamelCase_      =  BatchFeature({"input_features": features}     )

   lowerCamelCase_      =  self.pad(
       UpperCamelCase			,  padding=UpperCamelCase			,  max_length=UpperCamelCase			,  truncation=UpperCamelCase			,  pad_to_multiple_of=UpperCamelCase			,  return_attention_mask=UpperCamelCase			,  **UpperCamelCase			,  )

   # make sure list is in array format
   lowerCamelCase_      =  padded_inputs.get("input_features"     )
   if isinstance(input_features[0]			,  UpperCamelCase     ):
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for feature in input_features]

   lowerCamelCase_      =  padded_inputs.get("attention_mask"     )
   if attention_mask is not None:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.intaa     ) for array in attention_mask]

   # Utterance-level cepstral mean and variance normalization
   if self.do_ceptral_normalize:
    lowerCamelCase_      =  (
        np.array(UpperCamelCase			,  dtype=np.intaa     )
        if self._get_padding_strategies(UpperCamelCase			,  max_length=UpperCamelCase     ) is not PaddingStrategy.DO_NOT_PAD
        else None
    )
    lowerCamelCase_      =  self.normalize(
        padded_inputs["input_features"]			,  attention_mask=UpperCamelCase     )

   if return_tensors is not None:
    lowerCamelCase_      =  padded_inputs.convert_to_tensors(UpperCamelCase     )

   return padded_inputs







'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


a_  :				Any						= logging.get_logger(__name__)

a_  :				Optional[Any]						= {"""vocab_file""": """spiece.model"""}

a_  :				Tuple						= {
    """vocab_file""": {
        """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
    }
}





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase=False			,  UpperCamelCase=True			,  UpperCamelCase=False			,  UpperCamelCase="<s>"			,  UpperCamelCase="</s>"			,  UpperCamelCase="<unk>"			,  UpperCamelCase="<sep>"			,  UpperCamelCase="<pad>"			,  UpperCamelCase="<cls>"			,  UpperCamelCase="<mask>"			,  UpperCamelCase=["<eop>", "<eod>"]			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""
   lowerCamelCase_      =  AddedToken(UpperCamelCase			,  lstrip=UpperCamelCase			,  rstrip=UpperCamelCase     ) if isinstance(UpperCamelCase			,  UpperCamelCase     ) else mask_token

   lowerCamelCase_      =  {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=UpperCamelCase			,  remove_space=UpperCamelCase			,  keep_accents=UpperCamelCase			,  bos_token=UpperCamelCase			,  eos_token=UpperCamelCase			,  unk_token=UpperCamelCase			,  sep_token=UpperCamelCase			,  pad_token=UpperCamelCase			,  cls_token=UpperCamelCase			,  mask_token=UpperCamelCase			,  additional_special_tokens=UpperCamelCase			,  sp_model_kwargs=self.sp_model_kwargs			,  **UpperCamelCase			,  )

   lowerCamelCase_      =  3

   lowerCamelCase_      =  do_lower_case
   lowerCamelCase_      =  remove_space
   lowerCamelCase_      =  keep_accents
   lowerCamelCase_      =  vocab_file

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(UpperCamelCase     )

   try:
    import jieba
   except ModuleNotFoundError as error:
    raise error.__class__(
        "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
        "See https://pypi.org/project/jieba/ for installation."     )
   lowerCamelCase_      =  jieba
   lowerCamelCase_      =  str.maketrans(" \n"			,  "\u2582\u2583"     )






  @property
  # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
  def   snake_case		(    self     ):







   """simple docstring"""
   return len(self.sp_model     )






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  {self.convert_ids_to_tokens(UpperCamelCase     ): i for i in range(self.vocab_size     )}
   vocab.update(self.added_tokens_encoder     )
   return vocab






  def __getstate__(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  self.__dict__.copy()
   lowerCamelCase_      =  None
   return state






  def __setstate__(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  d

   # for backward compatibility
   if not hasattr(self			,  "sp_model_kwargs"     ):
    lowerCamelCase_      =  {}

   lowerCamelCase_      =  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
   self.sp_model.Load(self.vocab_file     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   if self.remove_space:
    lowerCamelCase_      =  " ".join(inputs.strip().split()     )
   else:
    lowerCamelCase_      =  inputs
   lowerCamelCase_      =  outputs.replace("``"			,  "\""     ).replace("''"			,  "\""     )

   if not self.keep_accents:
    lowerCamelCase_      =  unicodedata.normalize("NFKD"			,  UpperCamelCase     )
    lowerCamelCase_      =  "".join([c for c in outputs if not unicodedata.combining(UpperCamelCase     )]     )
   if self.do_lower_case:
    lowerCamelCase_      =  outputs.lower()

   return outputs






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.preprocess_text(UpperCamelCase     )
   lowerCamelCase_      =  self.sp_model.encode(UpperCamelCase			,  out_type=UpperCamelCase     )
   lowerCamelCase_      =  []
   for piece in pieces:
    if len(UpperCamelCase     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
     lowerCamelCase_      =  self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase			,  ""     )     )
     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
      if len(cur_pieces[0]     ) == 1:
       lowerCamelCase_      =  cur_pieces[1:]
      else:
       lowerCamelCase_      =  cur_pieces[0][1:]
     cur_pieces.append(piece[-1]     )
     new_pieces.extend(UpperCamelCase     )
    else:
     new_pieces.append(UpperCamelCase     )

   return new_pieces






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.PieceToId(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   return self.sp_model.IdToPiece(UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  "".join(UpperCamelCase     ).replace(UpperCamelCase			,  " "     ).strip()
   return out_string






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.sep_token_id]
   lowerCamelCase_      =  [self.cls_token_id]
   if token_ids_a is None:
    return token_ids_a + sep + cls
   return token_ids_a + sep + token_ids_a + sep + cls






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None			,  UpperCamelCase = False     ):







   """simple docstring"""

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=UpperCamelCase			,  token_ids_a=UpperCamelCase			,  already_has_special_tokens=UpperCamelCase     )

   if token_ids_a is not None:
    return ([0] * len(UpperCamelCase     )) + [1] + ([0] * len(UpperCamelCase     )) + [1, 1]
   return ([0] * len(UpperCamelCase     )) + [1, 1]






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  [self.sep_token_id]
   lowerCamelCase_      =  [2]

   if token_ids_a is None:
    return len(token_ids_a + sep     ) * [0] + cls_segment_id
   return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   if not os.path.isdir(UpperCamelCase     ):
    logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
    return
   lowerCamelCase_      =  os.path.join(
       UpperCamelCase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

   if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase     ) and os.path.isfile(self.vocab_file     ):
    copyfile(self.vocab_file			,  UpperCamelCase     )
   elif not os.path.isfile(self.vocab_file     ):
    with open(UpperCamelCase			,  "wb"     ) as fi:
     lowerCamelCase_      =  self.sp_model.serialized_model_proto()
     fi.write(UpperCamelCase     )

   return (out_vocab_file,)







  def   snake_case		(    self			,  *UpperCamelCase			,  **UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  super()._decode(*UpperCamelCase			,  **UpperCamelCase     )
   lowerCamelCase_      =  text.replace(" "			,  ""     ).replace("\u2582"			,  " "     ).replace("\u2583"			,  "\n"     )
   return text



