


"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


lowercase  		=			logging.get_logger(__name__)

lowercase  		=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase  		=			{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """allenai/longformer-base-4096""": (
            """https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase  		=			{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}


class 			SCREAMING_SNAKE_CASE_  (   _lowercase):




							'''simple docstring'''





							__magic_name__					:     Optional[int]				  =  VOCAB_FILES_NAMES
							__magic_name__					:     Union[str, Any]				  =  PRETRAINED_VOCAB_FILES_MAP
							__magic_name__					:     List[Any]				  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__magic_name__					:     List[Any]				  =  ['''input_ids''', '''attention_mask''']
							__magic_name__					:     Dict				  =  LongformerTokenizer




							def __init__(						self	,					lowerCamelCase__=None	,					lowerCamelCase__=None	,					lowerCamelCase__=None	,					lowerCamelCase__="replace"	,					lowerCamelCase__="<s>"	,					lowerCamelCase__="</s>"	,					lowerCamelCase__="</s>"	,					lowerCamelCase__="<s>"	,					lowerCamelCase__="<unk>"	,					lowerCamelCase__="<pad>"	,					lowerCamelCase__="<mask>"	,					lowerCamelCase__=False	,					lowerCamelCase__=True	,					**lowerCamelCase__	,					)   ->	List[str]:







								'''simple docstring'''




								super().__init__(
								    lowerCamelCase__	,					lowerCamelCase__	,					tokenizer_file=lowerCamelCase__	,					errors=lowerCamelCase__	,					bos_token=lowerCamelCase__	,					eos_token=lowerCamelCase__	,					sep_token=lowerCamelCase__	,					cls_token=lowerCamelCase__	,					unk_token=lowerCamelCase__	,					pad_token=lowerCamelCase__	,					mask_token=lowerCamelCase__	,					add_prefix_space=lowerCamelCase__	,					trim_offsets=lowerCamelCase__	,					**lowerCamelCase__	,					)

								snake_case__  :						Tuple		=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
								if pre_tok_state.get("add_prefix_space"	,					lowerCamelCase__) != add_prefix_space:
									snake_case__  :						Optional[Any]		=      getattr(lowerCamelCase__	,					pre_tok_state.pop("type"))
									snake_case__  :						Tuple		=      add_prefix_space
									snake_case__  :						Any		=      pre_tok_class(**lowerCamelCase__)

								snake_case__  :						Tuple		=      add_prefix_space

								snake_case__  :						Any		=      "post_processor"
								snake_case__  :						List[Any]		=      getattr(self.backend_tokenizer	,					lowerCamelCase__	,					lowerCamelCase__)
								if tokenizer_component_instance:
									snake_case__  :						List[Any]		=      json.loads(tokenizer_component_instance.__getstate__())

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
										snake_case__  :						int		=      tuple(state["sep"])
									if "cls" in state:
										snake_case__  :						str		=      tuple(state["cls"])

									snake_case__  :						Union[str, Any]		=      False

									if state.get("add_prefix_space"	,					lowerCamelCase__) != add_prefix_space:
										snake_case__  :						List[str]		=      add_prefix_space
										snake_case__  :						str		=      True

									if state.get("trim_offsets"	,					lowerCamelCase__) != trim_offsets:
										snake_case__  :						Union[str, Any]		=      trim_offsets
										snake_case__  :						int		=      True

									if changes_to_apply:
										snake_case__  :						Union[str, Any]		=      getattr(lowerCamelCase__	,					state.pop("type"))
										snake_case__  :						List[str]		=      component_class(**lowerCamelCase__)
										setattr(self.backend_tokenizer	,					lowerCamelCase__	,					lowerCamelCase__)




							@property
							def        UpperCAmelCase (						self)   ->	str:







								'''simple docstring'''




								if self._mask_token is None:
									if self.verbose:
										logger.error("Using mask_token, but it is not set yet.")
									return None
								return str(self._mask_token)




							@mask_token.setter
							def        UpperCAmelCase (						self	,					lowerCamelCase__)   ->	str:







								'''simple docstring'''




								snake_case__  :						Tuple		=      AddedToken(lowerCamelCase__	,					lstrip=lowerCamelCase__	,					rstrip=lowerCamelCase__) if isinstance(lowerCamelCase__	,					lowerCamelCase__) else value
								snake_case__  :						Tuple		=      value




							def        UpperCAmelCase (						self	,					*lowerCamelCase__	,					**lowerCamelCase__)   ->	BatchEncoding:







								'''simple docstring'''




								snake_case__  :						Optional[int]		=      kwargs.get("is_split_into_words"	,					lowerCamelCase__)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*lowerCamelCase__	,					**lowerCamelCase__)




							def        UpperCAmelCase (						self	,					*lowerCamelCase__	,					**lowerCamelCase__)   ->	BatchEncoding:







								'''simple docstring'''




								snake_case__  :						str		=      kwargs.get("is_split_into_words"	,					lowerCamelCase__)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*lowerCamelCase__	,					**lowerCamelCase__)




							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__ = None)   ->	Tuple[str]:







								'''simple docstring'''




								snake_case__  :						str		=      self._tokenizer.model.save(lowerCamelCase__	,					name=lowerCamelCase__)
								return tuple(lowerCamelCase__)




							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__=None)   ->	str:







								'''simple docstring'''




								snake_case__  :						Tuple		=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
									return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def        UpperCAmelCase (						self	,					lowerCamelCase__	,					lowerCamelCase__ = None)   ->	List[int]:







								'''simple docstring'''




								snake_case__  :						Optional[int]		=      [self.sep_token_id]
								snake_case__  :						List[str]		=      [self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]










import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:  str										=  logging.get_logger(__name__)

A_		:  Dict										=  {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}



class 		_lowercase		(							UpperCAmelCase__						):
   _UpperCAmelCase  						=      '''hubert'''

   def __init__(   self      :							Optional[Any]				, __lowerCAmelCase      :							str=32				, __lowerCAmelCase      :							int=768				, __lowerCAmelCase      :							Optional[int]=12				, __lowerCAmelCase      :							List[Any]=12				, __lowerCAmelCase      :							Optional[Any]=3072				, __lowerCAmelCase      :							Tuple="gelu"				, __lowerCAmelCase      :							Dict=0.1				, __lowerCAmelCase      :							List[Any]=0.1				, __lowerCAmelCase      :							str=0.1				, __lowerCAmelCase      :							Union[str, Any]=True				, __lowerCAmelCase      :							Tuple=0.0				, __lowerCAmelCase      :							Optional[int]=0.1				, __lowerCAmelCase      :							List[str]=0.1				, __lowerCAmelCase      :							Dict=0.0_2				, __lowerCAmelCase      :							Any=1E-5				, __lowerCAmelCase      :							Optional[Any]="group"				, __lowerCAmelCase      :							int="gelu"				, __lowerCAmelCase      :							Dict=(512, 512, 512, 512, 512, 512, 512)				, __lowerCAmelCase      :							Tuple=(5, 2, 2, 2, 2, 2, 2)				, __lowerCAmelCase      :							Tuple=(10, 3, 3, 3, 3, 2, 2)				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							Any=128				, __lowerCAmelCase      :							Tuple=16				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							int=True				, __lowerCAmelCase      :							List[str]=0.0_5				, __lowerCAmelCase      :							List[str]=10				, __lowerCAmelCase      :							int=2				, __lowerCAmelCase      :							List[Any]=0.0				, __lowerCAmelCase      :							Dict=10				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							Union[str, Any]="sum"				, __lowerCAmelCase      :							Dict=False				, __lowerCAmelCase      :							List[str]=False				, __lowerCAmelCase      :							int=256				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							str=1				, __lowerCAmelCase      :							Any=2				, **__lowerCAmelCase      :							Tuple				, )							->					Tuple:


    """simple docstring"""






    super().__init__(**__lowerCAmelCase				, pad_token_id=__lowerCAmelCase				, bos_token_id=__lowerCAmelCase				, eos_token_id=__lowerCAmelCase				)
    a							=		hidden_size
    a							=		feat_extract_norm
    a							=		feat_extract_activation
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		conv_bias
    a							=		num_conv_pos_embeddings
    a							=		num_conv_pos_embedding_groups
    a							=		len(self.conv_dim				)
    a							=		num_hidden_layers
    a							=		intermediate_size
    a							=		hidden_act
    a							=		num_attention_heads
    a							=		hidden_dropout
    a							=		attention_dropout
    a							=		activation_dropout
    a							=		feat_proj_layer_norm
    a							=		feat_proj_dropout
    a							=		final_dropout
    a							=		layerdrop
    a							=		layer_norm_eps
    a							=		initializer_range
    a							=		vocab_size
    a							=		do_stable_layer_norm
    a							=		use_weighted_layer_sum
    a							=		classifier_proj_size

    if (
        (len(self.conv_stride				) != self.num_feat_extract_layers)
        or (len(self.conv_kernel				) != self.num_feat_extract_layers)
        or (len(self.conv_dim				) != self.num_feat_extract_layers)
    ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         f""" {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,"""
         f""" `len(config.conv_kernel) = {len(self.conv_kernel				)}`."""				)

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    a							=		apply_spec_augment
    a							=		mask_time_prob
    a							=		mask_time_length
    a							=		mask_time_min_masks
    a							=		mask_feature_prob
    a							=		mask_feature_length
    a							=		mask_feature_min_masks

    # ctc loss
    a							=		ctc_loss_reduction
    a							=		ctc_zero_infinity




   @property
   def 		A				(   self      :							Tuple				)							->					Optional[Any]:


    """simple docstring"""






    return functools.reduce(operator.mul				, self.conv_stride				, 1				)

