





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE_						   =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_						   =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_						   =       {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE_						   =       {
    """google/realm-cc-news-pretrained-embedder""": 5_1_2,
    """google/realm-cc-news-pretrained-encoder""": 5_1_2,
    """google/realm-cc-news-pretrained-scorer""": 5_1_2,
    """google/realm-cc-news-pretrained-openqa""": 5_1_2,
    """google/realm-orqa-nq-openqa""": 5_1_2,
    """google/realm-orqa-nq-reader""": 5_1_2,
    """google/realm-orqa-wq-openqa""": 5_1_2,
    """google/realm-orqa-wq-reader""": 5_1_2,
}

SCREAMING_SNAKE_CASE_						   =       {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     __snake_case    :	Optional[Any]			=			VOCAB_FILES_NAMES
     __snake_case    :	List[Any]			=			PRETRAINED_VOCAB_FILES_MAP
     __snake_case    :	List[Any]			=			PRETRAINED_INIT_CONFIGURATION
     __snake_case    :	Optional[Any]			=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case    :	int			=			RealmTokenizer






     def __init__(							self						:    Union[str, Any]						,lowerCamelCase__						:    Dict=None						,lowerCamelCase__						:    List[Any]=None						,lowerCamelCase__						:    Union[str, Any]=True						,lowerCamelCase__						:    Dict="[UNK]"						,lowerCamelCase__						:    Any="[SEP]"						,lowerCamelCase__						:    Optional[Any]="[PAD]"						,lowerCamelCase__						:    Dict="[CLS]"						,lowerCamelCase__						:    Union[str, Any]="[MASK]"						,lowerCamelCase__						:    str=True						,lowerCamelCase__						:    Optional[Any]=None						,**lowerCamelCase__						:    List[str]						,)     ->		Any:




           '''simple docstring'''


           super().__init__(
               lowerCamelCase__						,tokenizer_file=lowerCamelCase__						,do_lower_case=lowerCamelCase__						,unk_token=lowerCamelCase__						,sep_token=lowerCamelCase__						,pad_token=lowerCamelCase__						,cls_token=lowerCamelCase__						,mask_token=lowerCamelCase__						,tokenize_chinese_chars=lowerCamelCase__						,strip_accents=lowerCamelCase__						,**lowerCamelCase__						,)

           SCREAMING_SNAKE_CASE		=						json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
           if (
               normalizer_state.get("""lowercase"""						,lowerCamelCase__       ) != do_lower_case
               or normalizer_state.get("""strip_accents"""						,lowerCamelCase__       ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""						,lowerCamelCase__       ) != tokenize_chinese_chars
           ):
                 SCREAMING_SNAKE_CASE		=						getattr(lowerCamelCase__						,normalizer_state.pop("""type"""       )       )
                 SCREAMING_SNAKE_CASE		=						do_lower_case
                 SCREAMING_SNAKE_CASE		=						strip_accents
                 SCREAMING_SNAKE_CASE		=						tokenize_chinese_chars
                 SCREAMING_SNAKE_CASE		=						normalizer_class(**lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						do_lower_case






     def 				SCREAMING_SNAKE_CASE__  (							self						:    str						,lowerCamelCase__						:    Optional[int]						,**lowerCamelCase__						:    Tuple       )     ->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						PaddingStrategy.MAX_LENGTH

           SCREAMING_SNAKE_CASE		=						text
           SCREAMING_SNAKE_CASE		=						kwargs.pop("""text_pair"""						,lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						kwargs.pop("""return_tensors"""						,lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						{
               """input_ids""": [],
               """attention_mask""": [],
               """token_type_ids""": [],
           }

           for idx, candidate_text in enumerate(lowerCamelCase__       ):
                 if batch_text_pair is not None:
                       SCREAMING_SNAKE_CASE		=						batch_text_pair[idx]
                 else:
                       SCREAMING_SNAKE_CASE		=						None

                 SCREAMING_SNAKE_CASE		=						super().__call__(lowerCamelCase__						,lowerCamelCase__						,return_tensors=lowerCamelCase__						,**lowerCamelCase__       )

                 SCREAMING_SNAKE_CASE		=						encoded_candidates.get("""input_ids"""       )
                 SCREAMING_SNAKE_CASE		=						encoded_candidates.get("""attention_mask"""       )
                 SCREAMING_SNAKE_CASE		=						encoded_candidates.get("""token_type_ids"""       )

                 if encoded_input_ids is not None:
                       output_data["input_ids"].append(lowerCamelCase__       )
                 if encoded_attention_mask is not None:
                       output_data["attention_mask"].append(lowerCamelCase__       )
                 if encoded_token_type_ids is not None:
                       output_data["token_type_ids"].append(lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						{key: item for key, item in output_data.items() if len(lowerCamelCase__       ) != 0}

           return BatchEncoding(lowerCamelCase__						,tensor_type=lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int						,lowerCamelCase__						:    Union[str, Any]						,lowerCamelCase__						:    str=None       )     ->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]						,lowerCamelCase__						:    Any						,lowerCamelCase__						:    str = None       )     ->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						[self.sep_token_id]
           SCREAMING_SNAKE_CASE		=						[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep       ) * [0]
           return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Any						,lowerCamelCase__						:    Any						,lowerCamelCase__						:    Any = None       )     ->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self._tokenizer.model.save(lowerCamelCase__						,name=lowerCamelCase__       )
           return tuple(lowerCamelCase__       )






'''simple docstring'''

from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


a_  :				int						= logging.get_logger(__name__)





class      snake_case    (    lowercase			):





  """simple docstring"""




  _lowerCamelCase		    =  ["input_features", "attention_mask"]






  def __init__(    self			,  UpperCamelCase=80			,  UpperCamelCase=1_6000			,  UpperCamelCase=80			,  UpperCamelCase=0.0			,  UpperCamelCase=True			,  UpperCamelCase=True			,  UpperCamelCase=True			,  **UpperCamelCase			,  ):







   """simple docstring"""
   super().__init__(feature_size=UpperCamelCase			,  sampling_rate=UpperCamelCase			,  padding_value=UpperCamelCase			,  **UpperCamelCase     )
   lowerCamelCase_      =  num_mel_bins
   lowerCamelCase_      =  do_ceptral_normalize
   lowerCamelCase_      =  normalize_means
   lowerCamelCase_      =  normalize_vars
   lowerCamelCase_      =  True






  def   snake_case		(    self			,  UpperCamelCase			,  ):







   """simple docstring"""
   lowerCamelCase_      =  waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
   lowerCamelCase_      =  torch.from_numpy(UpperCamelCase     ).unsqueeze(0     )
   lowerCamelCase_      =  ta_kaldi.fbank(UpperCamelCase			,  num_mel_bins=self.num_mel_bins			,  sample_frequency=self.sampling_rate     )
   return features.numpy()






  @staticmethod
  def   snake_case		(    UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase = True			,  UpperCamelCase = True			,  UpperCamelCase = 0.0			,  ):







   """simple docstring"""
   # make sure we normalize float32 arrays
   if normalize_means:
    lowerCamelCase_      =  x[:input_length].mean(axis=0     )
    lowerCamelCase_      =  np.subtract(UpperCamelCase			,  UpperCamelCase     )
   if normalize_vars:
    lowerCamelCase_      =  x[:input_length].std(axis=0     )
    lowerCamelCase_      =  np.divide(UpperCamelCase			,  UpperCamelCase     )

   if input_length < x.shape[0]:
    lowerCamelCase_      =  padding_value

   # make sure array is in float32
   lowerCamelCase_      =  x.astype(np.floataa     )

   return x






  def   snake_case		(    self			,  UpperCamelCase			,  UpperCamelCase = None     ):







   """simple docstring"""
   lowerCamelCase_      =  attention_mask.sum(-1     ) if attention_mask is not None else [x.shape[0] for x in input_features]
   return [
       self.utterance_cmvn(UpperCamelCase			,  UpperCamelCase			,  self.normalize_means			,  self.normalize_vars			,  self.padding_value     )
       for x, n in zip(UpperCamelCase			,  UpperCamelCase     )
   ]







  def __call__(    self			,  UpperCamelCase			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = False			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  UpperCamelCase = None			,  **UpperCamelCase			,  ):







   """simple docstring"""

   if sampling_rate is not None:
    if sampling_rate != self.sampling_rate:
     raise ValueError(
         f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
         f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
         f''' {self.sampling_rate} and not {sampling_rate}.'''     )
   else:
    logger.warning(
        "It is strongly recommended to pass the `sampling_rate` argument to this function. "
        "Failing to do so can result in silent errors that might be hard to debug."     )

   lowerCamelCase_      =  isinstance(UpperCamelCase			,  np.ndarray     ) and len(raw_speech.shape     ) > 1
   if is_batched_numpy and len(raw_speech.shape     ) > 2:
    raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''     )
   lowerCamelCase_      =  is_batched_numpy or (
       isinstance(UpperCamelCase			,  (list, tuple)     ) and (isinstance(raw_speech[0]			,  (np.ndarray, tuple, list)     ))
   )

   if is_batched:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for speech in raw_speech]
   elif not is_batched and not isinstance(UpperCamelCase			,  np.ndarray     ):
    lowerCamelCase_      =  np.asarray(UpperCamelCase			,  dtype=np.floataa     )
   elif isinstance(UpperCamelCase			,  np.ndarray     ) and raw_speech.dtype is np.dtype(np.floataa     ):
    lowerCamelCase_      =  raw_speech.astype(np.floataa     )

   # always return batch
   if not is_batched:
    lowerCamelCase_      =  [raw_speech]

   # extract fbank features
   lowerCamelCase_      =  [self._extract_fbank_features(UpperCamelCase     ) for waveform in raw_speech]

   # convert into correct format for padding
   lowerCamelCase_      =  BatchFeature({"input_features": features}     )

   lowerCamelCase_      =  self.pad(
       UpperCamelCase			,  padding=UpperCamelCase			,  max_length=UpperCamelCase			,  truncation=UpperCamelCase			,  pad_to_multiple_of=UpperCamelCase			,  return_attention_mask=UpperCamelCase			,  **UpperCamelCase			,  )

   # make sure list is in array format
   lowerCamelCase_      =  padded_inputs.get("input_features"     )
   if isinstance(input_features[0]			,  UpperCamelCase     ):
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.floataa     ) for feature in input_features]

   lowerCamelCase_      =  padded_inputs.get("attention_mask"     )
   if attention_mask is not None:
    lowerCamelCase_      =  [np.asarray(UpperCamelCase			,  dtype=np.intaa     ) for array in attention_mask]

   # Utterance-level cepstral mean and variance normalization
   if self.do_ceptral_normalize:
    lowerCamelCase_      =  (
        np.array(UpperCamelCase			,  dtype=np.intaa     )
        if self._get_padding_strategies(UpperCamelCase			,  max_length=UpperCamelCase     ) is not PaddingStrategy.DO_NOT_PAD
        else None
    )
    lowerCamelCase_      =  self.normalize(
        padded_inputs["input_features"]			,  attention_mask=UpperCamelCase     )

   if return_tensors is not None:
    lowerCamelCase_      =  padded_inputs.convert_to_tensors(UpperCamelCase     )

   return padded_inputs



