






"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase						:    Optional[Any]		=						logging.get_logger(__name__)

_UpperCamelCase						:    Optional[Any]		=						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCamelCase						:    List[Any]		=						{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
}

_UpperCamelCase						:    Optional[int]		=						{
    'roberta-base': 5_1_2,
    'roberta-large': 5_1_2,
    'roberta-large-mnli': 5_1_2,
    'distilroberta-base': 5_1_2,
    'roberta-base-openai-detector': 5_1_2,
    'roberta-large-openai-detector': 5_1_2,
}
@lru_cache()
def 		_SCREAMING_SNAKE_CASE					(       ):





  '''simple docstring'''





  lowercase				       =			(
      list(range(ord('!'       )						,   ord('~'       ) + 1       )       ) + list(range(ord('¡'       )						,   ord('¬'       ) + 1       )       ) + list(range(ord('®'       )						,   ord('ÿ'       ) + 1       )       )
  )
  lowercase				       =			bs[:]
  lowercase				       =			0
  for b in range(2**8       ):
    if b not in bs:
      bs.append(__snake_case       )
      cs.append(2**8 + n       )
      n += 1
  lowercase				       =			[chr(__snake_case       ) for n in cs]
  return dict(zip(__snake_case						,   __snake_case       )       )
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    Optional[int]       ):





  '''simple docstring'''





  lowercase				       =			set()
  lowercase				       =			word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)       )
    lowercase				       =			char
  return pairs



class 			a	(  a_      ):
    UpperCAmelCase_					:	List[str]      	=VOCAB_FILES_NAMES
    UpperCAmelCase_					:	Any      	=PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_					:	Tuple      	=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_					:	Dict      	=["input_ids", "attention_mask"]







    def __init__(     self				,						_lowerCamelCase				,						_lowerCamelCase				,						_lowerCamelCase="replace"				,						_lowerCamelCase="<s>"				,						_lowerCamelCase="</s>"				,						_lowerCamelCase="</s>"				,						_lowerCamelCase="<s>"				,						_lowerCamelCase="<unk>"				,						_lowerCamelCase="<pad>"				,						_lowerCamelCase="<mask>"				,						_lowerCamelCase=False				,						**_lowerCamelCase				,						):
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else bos_token
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else eos_token
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else sep_token
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else cls_token
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else unk_token
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      lowercase				       =			AddedToken(_lowerCamelCase				,						lstrip=_lowerCamelCase				,						rstrip=_lowerCamelCase			) if isinstance(_lowerCamelCase				,						_lowerCamelCase			) else mask_token

      super().__init__(
          errors=_lowerCamelCase				,						bos_token=_lowerCamelCase				,						eos_token=_lowerCamelCase				,						unk_token=_lowerCamelCase				,						sep_token=_lowerCamelCase				,						cls_token=_lowerCamelCase				,						pad_token=_lowerCamelCase				,						mask_token=_lowerCamelCase				,						add_prefix_space=_lowerCamelCase				,						**_lowerCamelCase				,						)

      with open(_lowerCamelCase				,						encoding='utf-8'			) as vocab_handle:
        lowercase				       =			json.load(_lowerCamelCase			)
      lowercase				       =			{v: k for k, v in self.encoder.items()}
      lowercase				       =			errors  # how to handle errors in decoding
      lowercase				       =			bytes_to_unicode()
      lowercase				       =			{v: k for k, v in self.byte_encoder.items()}
      with open(_lowerCamelCase				,						encoding='utf-8'			) as merges_handle:
        lowercase				       =			merges_handle.read().split('\n'			)[1:-1]
      lowercase				       =			[tuple(merge.split()			) for merge in bpe_merges]
      lowercase				       =			dict(zip(_lowerCamelCase				,						range(len(_lowerCamelCase			)			)			)			)
      lowercase				       =			{}
      lowercase				       =			add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      lowercase				       =			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'			)







    @property
    def      UpperCamelCase_ (     self			):
      return len(self.encoder			)







    def      UpperCamelCase_ (     self			):
      return dict(self.encoder				,						**self.added_tokens_encoder			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      if token in self.cache:
        return self.cache[token]
      lowercase				       =			tuple(_lowerCamelCase			)
      lowercase				       =			get_pairs(_lowerCamelCase			)

      if not pairs:
        return token

      while True:
        lowercase				       =			min(_lowerCamelCase				,						key=lambda _lowerCamelCase			: self.bpe_ranks.get(_lowerCamelCase				,						float('inf'			)			)			)
        if bigram not in self.bpe_ranks:
          break
        lowercase       ,	lowercase				       =			bigram
        lowercase				       =			[]
        lowercase				       =			0
        while i < len(_lowerCamelCase			):
          try:
            lowercase				       =			word.index(_lowerCamelCase				,						_lowerCamelCase			)
          except ValueError:
            new_word.extend(word[i:]			)
            break
          else:
            new_word.extend(word[i:j]			)
            lowercase				       =			j

          if word[i] == first and i < len(_lowerCamelCase			) - 1 and word[i + 1] == second:
            new_word.append(first + second			)
            i += 2
          else:
            new_word.append(word[i]			)
            i += 1
        lowercase				       =			tuple(_lowerCamelCase			)
        lowercase				       =			new_word
        if len(_lowerCamelCase			) == 1:
          break
        else:
          lowercase				       =			get_pairs(_lowerCamelCase			)
      lowercase				       =			' '.join(_lowerCamelCase			)
      lowercase				       =			word
      return word







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			[]
      for token in re.findall(self.pat				,						_lowerCamelCase			):
        lowercase				       =			''.join(
            self.byte_encoder[b] for b in token.encode('utf-8'			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase			).split(' '			)			)
      return bpe_tokens







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      return self.encoder.get(_lowerCamelCase				,						self.encoder.get(self.unk_token			)			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      return self.decoder.get(_lowerCamelCase			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			''.join(_lowerCamelCase			)
      lowercase				       =			bytearray([self.byte_decoder[c] for c in text]			).decode('utf-8'				,						errors=self.errors			)
      return text







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      if not os.path.isdir(_lowerCamelCase			):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
        return
      lowercase				       =			os.path.join(
          _lowerCamelCase				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
      lowercase				       =			os.path.join(
          _lowerCamelCase				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']			)

      with open(_lowerCamelCase				,						'w'				,						encoding='utf-8'			) as f:
        f.write(json.dumps(self.encoder				,						indent=2				,						sort_keys=_lowerCamelCase				,						ensure_ascii=_lowerCamelCase			) + '\n'			)

      lowercase				       =			0
      with open(_lowerCamelCase				,						'w'				,						encoding='utf-8'			) as writer:
        writer.write('#version: 0.2\n'			)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,						key=lambda _lowerCamelCase			: kv[1]			):
          if index != token_index:
            logger.warning(
                F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                ' Please check that the tokenizer is not corrupted!'			)
            lowercase				       =			token_index
          writer.write(' '.join(_lowerCamelCase			) + '\n'			)
          index += 1

      return vocab_file, merge_file







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      lowercase				       =			[self.cls_token_id]
      lowercase				       =			[self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None				,						_lowerCamelCase = False			):
      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=_lowerCamelCase				,						token_ids_a=_lowerCamelCase				,						already_has_special_tokens=_lowerCamelCase			)

      if token_ids_a is None:
        return [1] + ([0] * len(_lowerCamelCase			)) + [1]
      return [1] + ([0] * len(_lowerCamelCase			)) + [1, 1] + ([0] * len(_lowerCamelCase			)) + [1]







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase = None			):
      lowercase				       =			[self.sep_token_id]
      lowercase				       =			[self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]







    def      UpperCamelCase_ (     self				,						_lowerCamelCase				,						_lowerCamelCase=False				,						**_lowerCamelCase			):
      lowercase				       =			kwargs.pop('add_prefix_space'				,						self.add_prefix_space			)
      if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase			) > 0 and not text[0].isspace()):
        lowercase				       =			' ' + text
      return (text, kwargs)








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase							=      logging.get_logger(__name__)

_lowerCamelCase							=      {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}


class        UpperCAmelCase__				(							UpperCAmelCase__				):



     '''simple docstring'''
     _SCREAMING_SNAKE_CASE      :					Dict     					=    "hubert"
     def __init__(				self ,    _lowerCAmelCase=32 ,    _lowerCAmelCase=768 ,    _lowerCAmelCase=12 ,    _lowerCAmelCase=12 ,    _lowerCAmelCase=3_072 ,    _lowerCAmelCase="gelu" ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=True ,    _lowerCAmelCase=0.0 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.1 ,    _lowerCAmelCase=0.02 ,    _lowerCAmelCase=1E-5 ,    _lowerCAmelCase="group" ,    _lowerCAmelCase="gelu" ,    _lowerCAmelCase=(512, 512, 512, 512, 512, 512, 512) ,    _lowerCAmelCase=(5, 2, 2, 2, 2, 2, 2) ,    _lowerCAmelCase=(10, 3, 3, 3, 3, 2, 2) ,    _lowerCAmelCase=False ,    _lowerCAmelCase=128 ,    _lowerCAmelCase=16 ,    _lowerCAmelCase=False ,    _lowerCAmelCase=True ,    _lowerCAmelCase=0.05 ,    _lowerCAmelCase=10 ,    _lowerCAmelCase=2 ,    _lowerCAmelCase=0.0 ,    _lowerCAmelCase=10 ,    _lowerCAmelCase=0 ,    _lowerCAmelCase="sum" ,    _lowerCAmelCase=False ,    _lowerCAmelCase=False ,    _lowerCAmelCase=256 ,    _lowerCAmelCase=0 ,    _lowerCAmelCase=1 ,    _lowerCAmelCase=2 ,    **_lowerCAmelCase ,    ):
            super().__init__(**_lowerCAmelCase ,    pad_token_id=_lowerCAmelCase ,    bos_token_id=_lowerCAmelCase ,    eos_token_id=_lowerCAmelCase	)
            a			   =hidden_size
            a			   =feat_extract_norm
            a			   =feat_extract_activation
            a			   =list(_lowerCAmelCase	)
            a			   =list(_lowerCAmelCase	)
            a			   =list(_lowerCAmelCase	)
            a			   =conv_bias
            a			   =num_conv_pos_embeddings
            a			   =num_conv_pos_embedding_groups
            a			   =len(self.conv_dim	)
            a			   =num_hidden_layers
            a			   =intermediate_size
            a			   =hidden_act
            a			   =num_attention_heads
            a			   =hidden_dropout
            a			   =attention_dropout
            a			   =activation_dropout
            a			   =feat_proj_layer_norm
            a			   =feat_proj_dropout
            a			   =final_dropout
            a			   =layerdrop
            a			   =layer_norm_eps
            a			   =initializer_range
            a			   =vocab_size
            a			   =do_stable_layer_norm
            a			   =use_weighted_layer_sum
            a			   =classifier_proj_size

            if (
                (len(self.conv_stride	) != self.num_feat_extract_layers)
                or (len(self.conv_kernel	) != self.num_feat_extract_layers)
                or (len(self.conv_dim	) != self.num_feat_extract_layers)
            ):
                   raise ValueError(
                       """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                       """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                       F''' {len(self.conv_dim	)}`, `len(config.conv_stride) = {len(self.conv_stride	)}`,'''
                       F''' `len(config.conv_kernel) = {len(self.conv_kernel	)}`.'''	)

            # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
            a			   =apply_spec_augment
            a			   =mask_time_prob
            a			   =mask_time_length
            a			   =mask_time_min_masks
            a			   =mask_feature_prob
            a			   =mask_feature_length
            a			   =mask_feature_min_masks

            # ctc loss
            a			   =ctc_loss_reduction
            a			   =ctc_zero_infinity






     @property
     def        lowerCAmelCase__  (				self	):
            return functools.reduce(operator.mul ,    self.conv_stride ,    1	)
