
"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_big_bird import BigBirdTokenizer
else:
					UpperCAmelCase__      :    Union[str, Any]    							= None

UpperCAmelCase__      :    str    							= logging.get_logger(__name__)
UpperCAmelCase__      :    Optional[Any]    							= {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase__      :    Optional[int]    							= {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    },
    'tokenizer_file': {
        'google/bigbird-roberta-base': (
            'https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'
        ),
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase__      :    Optional[int]    							= {
    'google/bigbird-roberta-base': 4_0_9_6,
    'google/bigbird-roberta-large': 4_0_9_6,
    'google/bigbird-base-trivia-itc': 4_0_9_6,
}


UpperCAmelCase__      :    Union[str, Any]    							= '▁'



class 	lowerCAmelCase_						(a__  ):


				"""simple docstring"""



				__UpperCamelCase      :						Dict					       =   VOCAB_FILES_NAMES
				__UpperCamelCase      :						List[str]					       =   PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase      :						str					       =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase      :						Optional[Any]					       =   BigBirdTokenizer
				__UpperCamelCase      :						int					       =   ['''input_ids''', '''attention_mask''']
				__UpperCamelCase      :						List[int]					       =   []



				def __init__(self   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__="<unk>"   ,       SCREAMING_SNAKE_CASE__="<s>"   ,       SCREAMING_SNAKE_CASE__="</s>"   ,       SCREAMING_SNAKE_CASE__="<pad>"   ,       SCREAMING_SNAKE_CASE__="[SEP]"   ,       SCREAMING_SNAKE_CASE__="[MASK]"   ,       SCREAMING_SNAKE_CASE__="[CLS]"   ,       **SCREAMING_SNAKE_CASE__   ,       )					->		Any:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else bos_token
											SCREAMING_SNAKE_CASE__						:       Dict           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else eos_token
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else unk_token
											SCREAMING_SNAKE_CASE__						:       Any           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else pad_token
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else cls_token
											SCREAMING_SNAKE_CASE__						:       int           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else sep_token

											# Mask token behave like a normal word, i.e. include the space before it
											SCREAMING_SNAKE_CASE__						:       List[str]           =   AddedToken(SCREAMING_SNAKE_CASE__   ,       lstrip=SCREAMING_SNAKE_CASE__   ,       rstrip=SCREAMING_SNAKE_CASE__    ) if isinstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    ) else mask_token

											super().__init__(
											    SCREAMING_SNAKE_CASE__   ,       tokenizer_file=SCREAMING_SNAKE_CASE__   ,       bos_token=SCREAMING_SNAKE_CASE__   ,       eos_token=SCREAMING_SNAKE_CASE__   ,       unk_token=SCREAMING_SNAKE_CASE__   ,       sep_token=SCREAMING_SNAKE_CASE__   ,       pad_token=SCREAMING_SNAKE_CASE__   ,       cls_token=SCREAMING_SNAKE_CASE__   ,       mask_token=SCREAMING_SNAKE_CASE__   ,       **SCREAMING_SNAKE_CASE__   ,       )

											SCREAMING_SNAKE_CASE__						:       Any           =   vocab_file
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   False if not self.vocab_file else True



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		List[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Dict           =   [self.sep_token_id]
											SCREAMING_SNAKE_CASE__						:       str           =   [self.cls_token_id]
											if token_ids_a is None:
																		return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None   ,       SCREAMING_SNAKE_CASE__ = False    )					->		List[int]:



											"""simple docstring"""




											if already_has_special_tokens:
																		if token_ids_a is not None:
																									raise ValueError(
																									    """You should not supply a second sequence if the provided sequence of """
																									    """ids is already formatted with special tokens for the model."""    )
																		return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

											if token_ids_a is None:
																		return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]
											return [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__    )) + [1]



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		List[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   [self.sep_token_id]
											SCREAMING_SNAKE_CASE__						:       int           =   [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__ = None    )					->		Tuple[str]:



											"""simple docstring"""




											if not self.can_save_slow_tokenizer:
																		raise ValueError(
																		    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																		    """tokenizer."""    )

											if not os.path.isdir(SCREAMING_SNAKE_CASE__    ):
																		logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
																		return
											SCREAMING_SNAKE_CASE__						:       Dict           =   os.path.join(
											    SCREAMING_SNAKE_CASE__   ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(SCREAMING_SNAKE_CASE__    ):
																		copyfile(self.vocab_file   ,       SCREAMING_SNAKE_CASE__    )

											return (out_vocab_file,)






"""simple docstring"""




import mpmath  # for roots of unity
import numpy as np



class 	lowerCAmelCase_						:


				"""simple docstring"""



				def __init__(self   ,       SCREAMING_SNAKE_CASE__=None   ,       SCREAMING_SNAKE_CASE__=None    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   list(poly_a or [0]    )[:]
											SCREAMING_SNAKE_CASE__						:       Tuple           =   list(poly_b or [0]    )[:]

											# Remove leading zero coefficients
											while self.polyA[-1] == 0:
																		self.polyA.pop()
											SCREAMING_SNAKE_CASE__						:       int           =   len(self.polyA    )

											while self.polyB[-1] == 0:
																		self.polyB.pop()
											SCREAMING_SNAKE_CASE__						:       List[str]           =   len(self.polyB    )

											# Add 0 to make lengths equal a power of 2
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   int(
											    2 ** np.ceil(np.loga(len(self.polyA    ) + len(self.polyB    ) - 1    )    )    )

											while len(self.polyA    ) < self.c_max_length:
																		self.polyA.append(0    )
											while len(self.polyB    ) < self.c_max_length:
																		self.polyB.append(0    )
											# A complex root used for the fourier transform
											SCREAMING_SNAKE_CASE__						:       List[str]           =   complex(mpmath.root(x=1   ,       n=self.c_max_length   ,       k=1    )    )

											# The product
											SCREAMING_SNAKE_CASE__						:       Tuple           =   self.__multiply()



				def 	__magic_name__    (self   ,       SCREAMING_SNAKE_CASE__    )					->		Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       str           =   [[x] for x in self.polyA] if which == """A""" else [[x] for x in self.polyB]
											# Corner case
											if len(SCREAMING_SNAKE_CASE__    ) <= 1:
																		return dft[0]
											#
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   self.c_max_length // 2
											while next_ncol > 0:
																		SCREAMING_SNAKE_CASE__						:       Any           =   [[] for i in range(SCREAMING_SNAKE_CASE__    )]
																		SCREAMING_SNAKE_CASE__						:       Tuple           =   self.root**next_ncol

																		# First half of next step
																		SCREAMING_SNAKE_CASE__						:       str           =   1
																		for j in range(self.c_max_length // (next_ncol * 2)    ):
																									for i in range(SCREAMING_SNAKE_CASE__    ):
																																new_dft[i].append(dft[i][j] + current_root * dft[i + next_ncol][j]    )
																									current_root *= root
																		# Second half of next step
																		SCREAMING_SNAKE_CASE__						:       int           =   1
																		for j in range(self.c_max_length // (next_ncol * 2)    ):
																									for i in range(SCREAMING_SNAKE_CASE__    ):
																																new_dft[i].append(dft[i][j] - current_root * dft[i + next_ncol][j]    )
																									current_root *= root
																		# Update
																		SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   new_dft
																		SCREAMING_SNAKE_CASE__						:       Tuple           =   next_ncol // 2
											return dft[0]



				def 	__magic_name__    (self    )					->		int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   self.__dft("""A"""    )
											SCREAMING_SNAKE_CASE__						:       Dict           =   self.__dft("""B"""    )
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   [[dft_a[i] * dft_b[i] for i in range(self.c_max_length    )]]
											del dft_a
											del dft_b

											# Corner Case
											if len(inverce_c[0]    ) <= 1:
																		return inverce_c[0]
											# Inverse DFT
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   2
											while next_ncol <= self.c_max_length:
																		SCREAMING_SNAKE_CASE__						:       List[str]           =   [[] for i in range(SCREAMING_SNAKE_CASE__    )]
																		SCREAMING_SNAKE_CASE__						:       Tuple           =   self.root ** (next_ncol // 2)
																		SCREAMING_SNAKE_CASE__						:       Any           =   1
																		# First half of next step
																		for j in range(self.c_max_length // next_ncol    ):
																									for i in range(next_ncol // 2    ):
																																# Even positions
																																new_inverse_c[i].append(
																																    (
																																        inverce_c[i][j]
																																        + inverce_c[i][j + self.c_max_length // next_ncol]
																																    )
																																    / 2    )
																																# Odd positions
																																new_inverse_c[i + next_ncol // 2].append(
																																    (
																																        inverce_c[i][j]
																																        - inverce_c[i][j + self.c_max_length // next_ncol]
																																    )
																																    / (2 * current_root)    )
																									current_root *= root
																		# Update
																		SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   new_inverse_c
																		next_ncol *= 2
											# Unpack
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   [round(x[0].real   ,       8    ) + round(x[0].imag   ,       8    ) * 1j for x in inverce_c]

											# Remove leading 0's
											while inverce_c[-1] == 0:
																		inverce_c.pop()
											return inverce_c



				def __str__(self    )					->		List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[str]           =   """A = """ + """ + """.join(
											    F'''{coef}*x^{i}''' for coef, i in enumerate(self.polyA[: self.len_A]    )    )
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   """B = """ + """ + """.join(
											    F'''{coef}*x^{i}''' for coef, i in enumerate(self.polyB[: self.len_B]    )    )
											SCREAMING_SNAKE_CASE__						:       int           =   """A*B = """ + """ + """.join(
											    F'''{coef}*x^{i}''' for coef, i in enumerate(self.product    )    )

											return F'''{a}\n{b}\n{c}'''


# Unit tests
if __name__ == "__main__":
					import doctest

					doctest.testmod()




