



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

_snake_case		=  logging.get_logger(__name__)


_snake_case		=  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
    "tokenizer_config_file": "tokenizer_config.json",
}

_snake_case		=  {
    "vocab_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"},
    "merges_file": {"facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"},
    "tokenizer_config_file": {
        "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"
    },
}

_snake_case		=  {"facebook/blenderbot-3B": 128}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 				A				(					):





							'''simple docstring'''




							_lowerCAmelCase     :							Union[str, Any]    =    (
							    list(range(ord("!"  )				, ord("~"  ) + 1  )  ) + list(range(ord("¡"  )				, ord("¬"  ) + 1  )  ) + list(range(ord("®"  )				, ord("ÿ"  ) + 1  )  )
							)
							_lowerCAmelCase     :							List[str]    =    bs[:]
							_lowerCAmelCase     :							int    =    0
							for b in range(2**8  ):
														if b not in bs:
																					bs.append(_lowerCamelCase  )
																					cs.append(2**8 + n  )
																					n += 1
							_lowerCAmelCase     :							Optional[Any]    =    [chr(_lowerCamelCase  ) for n in cs]
							return dict(zip(_lowerCamelCase				, _lowerCamelCase  )  )






def 				A				(					_lowerCamelCase  ):





							'''simple docstring'''




							_lowerCAmelCase     :							List[Any]    =    set()
							_lowerCAmelCase     :							int    =    word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)  )
														_lowerCAmelCase     :							str    =    char
							return pairs



class   UpperCAmelCase_    (       a):
							lowerCamelCase__   			=		VOCAB_FILES_NAMES
							lowerCamelCase__   			=		PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase__   			=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase__   			=		['input_ids', 'attention_mask']







							def __init__(   self,			__a,			__a,			__a="replace",			__a="<s>",			__a="</s>",			__a="</s>",			__a="<s>",			__a="<unk>",			__a="<pad>",			__a="<mask>",			__a=False,			**__a,			):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else bos_token
														_lowerCAmelCase     :							int    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else eos_token
														_lowerCAmelCase     :							Union[str, Any]    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else sep_token
														_lowerCAmelCase     :							Any    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else cls_token
														_lowerCAmelCase     :							Dict    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else unk_token
														_lowerCAmelCase     :							List[str]    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else pad_token

														# Mask token behave like a normal word, i.e. include the space before it
														_lowerCAmelCase     :							Union[str, Any]    =    AddedToken(__a,			lstrip=__a,			rstrip=__a) if isinstance(__a,			__a) else mask_token

														super().__init__(
														    errors=__a,			bos_token=__a,			eos_token=__a,			unk_token=__a,			sep_token=__a,			cls_token=__a,			pad_token=__a,			mask_token=__a,			add_prefix_space=__a,			**__a,			)

														with open(__a,			encoding="utf-8") as vocab_handle:
																					_lowerCAmelCase     :							int    =    json.load(__a)
														_lowerCAmelCase     :							List[Any]    =    {v: k for k, v in self.encoder.items()}
														_lowerCAmelCase     :							Any    =    errors  # how to handle errors in decoding
														_lowerCAmelCase     :							int    =    bytes_to_unicode()
														_lowerCAmelCase     :							Dict    =    {v: k for k, v in self.byte_encoder.items()}
														with open(__a,			encoding="utf-8") as merges_handle:
																					_lowerCAmelCase     :							int    =    merges_handle.read().split("\n")[1:-1]
														_lowerCAmelCase     :							Any    =    [tuple(merge.split()) for merge in bpe_merges]
														_lowerCAmelCase     :							List[str]    =    dict(zip(__a,			range(len(__a))))
														_lowerCAmelCase     :							Optional[int]    =    {}
														_lowerCAmelCase     :							int    =    add_prefix_space

														# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
														_lowerCAmelCase     :							Tuple    =    re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")







							@property
							# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
							def    snake_case__      (   self):

														'''simple docstring'''

														return len(self.encoder)







							def    snake_case__      (   self):

														'''simple docstring'''

														return dict(self.encoder,			**self.added_tokens_encoder)







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														if token in self.cache:
																					return self.cache[token]
														_lowerCAmelCase     :							Any    =    tuple(__a)
														_lowerCAmelCase     :							List[str]    =    get_pairs(__a)

														if not pairs:
																					return token

														while True:
																					_lowerCAmelCase     :							List[Any]    =    min(__a,			key=lambda __a: self.bpe_ranks.get(__a,			float("inf")))
																					if bigram not in self.bpe_ranks:
																												break
																					_lowerCAmelCase   ,    _lowerCAmelCase     :							str    =    bigram
																					_lowerCAmelCase     :							Any    =    []
																					_lowerCAmelCase     :							Any    =    0
																					while i < len(__a):
																												try:
																																			_lowerCAmelCase     :							List[str]    =    word.index(__a,			__a)
																												except ValueError:
																																			new_word.extend(word[i:])
																																			break
																												else:
																																			new_word.extend(word[i:j])
																																			_lowerCAmelCase     :							Tuple    =    j

																												if word[i] == first and i < len(__a) - 1 and word[i + 1] == second:
																																			new_word.append(first + second)
																																			i += 2
																												else:
																																			new_word.append(word[i])
																																			i += 1
																					_lowerCAmelCase     :							int    =    tuple(__a)
																					_lowerCAmelCase     :							List[Any]    =    new_word
																					if len(__a) == 1:
																												break
																					else:
																												_lowerCAmelCase     :							Any    =    get_pairs(__a)
														_lowerCAmelCase     :							Union[str, Any]    =    " ".join(__a)
														_lowerCAmelCase     :							List[str]    =    word
														return word







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    []
														for token in re.findall(self.pat,			__a):
																					_lowerCAmelCase     :							int    =    "".join(
																					    self.byte_encoder[b] for b in token.encode("utf-8"))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__a).split(" "))
														return bpe_tokens







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														return self.encoder.get(__a,			self.encoder.get(self.unk_token))







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														return self.decoder.get(__a)







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							int    =    "".join(__a)
														_lowerCAmelCase     :							Optional[Any]    =    bytearray([self.byte_decoder[c] for c in text]).decode("utf-8",			errors=self.errors)
														return text







							def    snake_case__      (   self,			__a,			__a = None):

														'''simple docstring'''

														if not os.path.isdir(__a):
																					logger.error(f"Vocabulary path ({save_directory}) should be a directory")
																					return
														_lowerCAmelCase     :							int    =    os.path.join(
														    __a,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
														_lowerCAmelCase     :							List[str]    =    os.path.join(
														    __a,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

														with open(__a,			"w",			encoding="utf-8") as f:
																					f.write(json.dumps(self.encoder,			indent=2,			sort_keys=__a,			ensure_ascii=__a) + "\n")

														_lowerCAmelCase     :							int    =    0
														with open(__a,			"w",			encoding="utf-8") as writer:
																					writer.write("#version: 0.2\n")
																					for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),			key=lambda __a: kv[1]):
																												if index != token_index:
																																			logger.warning(
																																			    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																																			    " Please check that the tokenizer is not corrupted!")
																																			_lowerCAmelCase     :							Dict    =    token_index
																												writer.write(" ".join(__a) + "\n")
																												index += 1

														return vocab_file, merge_file







							def    snake_case__      (   self,			__a,			__a = None,			__a = False):

														'''simple docstring'''

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__a,			token_ids_a=__a,			already_has_special_tokens=__a)

														if token_ids_a is None:
																					return [1] + ([0] * len(__a)) + [1]
														return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a)) + [1]







							def    snake_case__      (   self,			__a,			__a = None):

														'''simple docstring'''

														_lowerCAmelCase     :							Any    =    [self.sep_token_id]
														_lowerCAmelCase     :							int    =    [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]







							def    snake_case__      (   self,			__a,			__a=False,			**__a):

														'''simple docstring'''

														_lowerCAmelCase     :							str    =    kwargs.pop("add_prefix_space",			self.add_prefix_space)
														if (is_split_into_words or add_prefix_space) and (len(__a) > 0 and not text[0].isspace()):
																					_lowerCAmelCase     :							Tuple    =    " " + text
														return (text, kwargs)







							def    snake_case__      (   self,			__a,			__a = None):

														'''simple docstring'''

														return token_ids_a + [self.eos_token_id]







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							Tuple    =    []
														for is_user, text in conversation.iter_texts():
																					if is_user:
																												# We need to space prefix as it's being done within blenderbot
																												inputs.append(" " + text)
																					else:
																												# Generated responses should contain them already.
																												inputs.append(__a)

														_lowerCAmelCase     :							Optional[Any]    =    "  ".join(__a)
														_lowerCAmelCase     :							Tuple    =    self.encode(__a)
														if len(__a) > self.model_max_length:
																					_lowerCAmelCase     :							str    =    input_ids[-self.model_max_length :]
																					logger.warning(f"Trimmed input from conversation as it was longer than {self.model_max_length} tokens.")
														return input_ids





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case		=  logging.get_logger(__name__)

_snake_case		=  {
    "facebook/wav2vec2-base-960h": "https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json",
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}



class   UpperCAmelCase_    (       a):
							lowerCamelCase__   			=		'wav2vec2'







							def __init__(   self,			__a=32,			__a=768,			__a=12,			__a=12,			__a=3072,			__a="gelu",			__a=0.1,			__a=0.1,			__a=0.1,			__a=0.0,			__a=0.0,			__a=0.1,			__a=0.1,			__a=0.02,			__a=1E-5,			__a="group",			__a="gelu",			__a=(512, 512, 512, 512, 512, 512, 512),			__a=(5, 2, 2, 2, 2, 2, 2),			__a=(10, 3, 3, 3, 3, 2, 2),			__a=False,			__a=128,			__a=16,			__a=False,			__a=True,			__a=0.05,			__a=10,			__a=2,			__a=0.0,			__a=10,			__a=0,			__a=320,			__a=2,			__a=0.1,			__a=100,			__a=256,			__a=256,			__a=0.1,			__a="sum",			__a=False,			__a=False,			__a=256,			__a=(512, 512, 512, 512, 1500),			__a=(5, 3, 3, 1, 1),			__a=(1, 2, 3, 1, 1),			__a=512,			__a=0,			__a=1,			__a=2,			__a=False,			__a=3,			__a=2,			__a=3,			__a=None,			__a=None,			**__a,			):

														'''simple docstring'''

														super().__init__(**__a,			pad_token_id=__a,			bos_token_id=__a,			eos_token_id=__a)
														_lowerCAmelCase     :							str    =    hidden_size
														_lowerCAmelCase     :							Optional[int]    =    feat_extract_norm
														_lowerCAmelCase     :							Union[str, Any]    =    feat_extract_activation
														_lowerCAmelCase     :							Optional[Any]    =    list(__a)
														_lowerCAmelCase     :							List[str]    =    list(__a)
														_lowerCAmelCase     :							str    =    list(__a)
														_lowerCAmelCase     :							List[str]    =    conv_bias
														_lowerCAmelCase     :							str    =    num_conv_pos_embeddings
														_lowerCAmelCase     :							List[Any]    =    num_conv_pos_embedding_groups
														_lowerCAmelCase     :							str    =    len(self.conv_dim)
														_lowerCAmelCase     :							List[str]    =    num_hidden_layers
														_lowerCAmelCase     :							str    =    intermediate_size
														_lowerCAmelCase     :							Any    =    hidden_act
														_lowerCAmelCase     :							int    =    num_attention_heads
														_lowerCAmelCase     :							Optional[Any]    =    hidden_dropout
														_lowerCAmelCase     :							List[str]    =    attention_dropout
														_lowerCAmelCase     :							Tuple    =    activation_dropout
														_lowerCAmelCase     :							int    =    feat_proj_dropout
														_lowerCAmelCase     :							List[str]    =    final_dropout
														_lowerCAmelCase     :							int    =    layerdrop
														_lowerCAmelCase     :							int    =    layer_norm_eps
														_lowerCAmelCase     :							Union[str, Any]    =    initializer_range
														_lowerCAmelCase     :							str    =    vocab_size
														_lowerCAmelCase     :							Optional[Any]    =    do_stable_layer_norm
														_lowerCAmelCase     :							Any    =    use_weighted_layer_sum

														if (
														    (len(self.conv_stride) != self.num_feat_extract_layers)
														    or (len(self.conv_kernel) != self.num_feat_extract_layers)
														    or (len(self.conv_dim) != self.num_feat_extract_layers)
														):
																					raise ValueError(
																					    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
																					    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
																					    f" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"
																					    f" `len(config.conv_kernel) = {len(self.conv_kernel)}`.")

														# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
														_lowerCAmelCase     :							str    =    apply_spec_augment
														_lowerCAmelCase     :							Optional[Any]    =    mask_time_prob
														_lowerCAmelCase     :							Optional[int]    =    mask_time_length
														_lowerCAmelCase     :							List[str]    =    mask_time_min_masks
														_lowerCAmelCase     :							Optional[int]    =    mask_feature_prob
														_lowerCAmelCase     :							Optional[int]    =    mask_feature_length
														_lowerCAmelCase     :							List[str]    =    mask_feature_min_masks

														# parameters for pretraining with codevector quantized representations
														_lowerCAmelCase     :							Union[str, Any]    =    num_codevectors_per_group
														_lowerCAmelCase     :							str    =    num_codevector_groups
														_lowerCAmelCase     :							Optional[int]    =    contrastive_logits_temperature
														_lowerCAmelCase     :							Optional[int]    =    feat_quantizer_dropout
														_lowerCAmelCase     :							Optional[int]    =    num_negatives
														_lowerCAmelCase     :							Union[str, Any]    =    codevector_dim
														_lowerCAmelCase     :							Any    =    proj_codevector_dim
														_lowerCAmelCase     :							Optional[int]    =    diversity_loss_weight

														# ctc loss
														_lowerCAmelCase     :							Tuple    =    ctc_loss_reduction
														_lowerCAmelCase     :							Tuple    =    ctc_zero_infinity

														# adapter
														_lowerCAmelCase     :							List[Any]    =    add_adapter
														_lowerCAmelCase     :							List[str]    =    adapter_kernel_size
														_lowerCAmelCase     :							str    =    adapter_stride
														_lowerCAmelCase     :							List[str]    =    num_adapter_layers
														_lowerCAmelCase     :							str    =    output_hidden_size or hidden_size
														_lowerCAmelCase     :							Tuple    =    adapter_attn_dim

														# SequenceClassification-specific parameter. Feel free to ignore for other classes.
														_lowerCAmelCase     :							str    =    classifier_proj_size

														# XVector-specific parameters. Feel free to ignore for other classes.
														_lowerCAmelCase     :							str    =    list(__a)
														_lowerCAmelCase     :							Union[str, Any]    =    list(__a)
														_lowerCAmelCase     :							List[str]    =    list(__a)
														_lowerCAmelCase     :							Tuple    =    xvector_output_dim







							@property
							def    snake_case__      (   self):

														'''simple docstring'''

														return functools.reduce(operator.mul,			self.conv_stride,			1)
