






import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase  							=					logging.get_logger(__name__)

lowercase  							=					{'vocab_file': 'spiece.model'}

lowercase  							=					{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

lowercase  							=					{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}




class  UpperCamelCase_      (			A__				):



	'''simple docstring'''




	lowerCAmelCase													= VOCAB_FILES_NAMES
	lowerCAmelCase													= PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase													= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase													= ['''input_ids''', '''attention_mask''']
	lowerCAmelCase													= []




	def __init__(       self       ,			a       ,			a="<unk>"       ,			a="<s>"       ,			a="</s>"       ,			a="<pad>"       ,			a="[SEP]"       ,			a="[MASK]"       ,			a="[CLS]"       ,			a = None       ,			**a       ,			)       ->    None:
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else bos_token
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else eos_token
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else unk_token
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else pad_token
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else cls_token
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else sep_token

								# Mask token behave like a normal word, i.e. include the space before it
								snake_case_				=							AddedToken(lowerCAmelCase__       ,			lstrip=lowerCAmelCase__       ,			rstrip=lowerCAmelCase__					) if isinstance(lowerCAmelCase__       ,			lowerCAmelCase__					) else mask_token

								snake_case_				=							{} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=lowerCAmelCase__       ,			eos_token=lowerCAmelCase__       ,			unk_token=lowerCAmelCase__       ,			pad_token=lowerCAmelCase__       ,			sep_token=lowerCAmelCase__       ,			mask_token=lowerCAmelCase__       ,			cls_token=lowerCAmelCase__       ,			sp_model_kwargs=self.sp_model_kwargs       ,			**lowerCAmelCase__       ,			)

								snake_case_				=							vocab_file

								snake_case_				=							spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(lowerCAmelCase__					)




	@property
	def 							_UpperCamelCase			(       self					)       ->    List[str]:
								return self.sp_model.get_piece_size()




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_				=							{self.convert_ids_to_tokens(lowerCAmelCase__					): i for i in range(self.vocab_size					)}
								vocab.update(self.added_tokens_encoder					)
								return vocab




	def __getstate__(       self					)       ->    List[Any]:
								snake_case_				=							self.__dict__.copy()
								snake_case_				=							None
								return state




	def __setstate__(       self       ,			a					)       ->    Any:
								snake_case_				=							d

								# for backward compatibility
								if not hasattr(self       ,			'sp_model_kwargs'					):
															snake_case_				=							{}

								snake_case_				=							spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(self.vocab_file					)




	def 							_UpperCamelCase			(       self       ,			a					)       ->    List[str]:
								return self.sp_model.encode(lowerCAmelCase__       ,			out_type=lowerCAmelCase__					)




	def 							_UpperCamelCase			(       self       ,			a					)       ->    List[Any]:
								return self.sp_model.piece_to_id(lowerCAmelCase__					)




	def 							_UpperCamelCase			(       self       ,			a					)       ->    Dict:
								snake_case_				=							self.sp_model.IdToPiece(lowerCAmelCase__					)
								return token




	def 							_UpperCamelCase			(       self       ,			a					)       ->    Union[str, Any]:
								snake_case_				=							[]
								snake_case_				=							''''''
								snake_case_				=							False
								for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																						if not prev_is_special:
																													out_string += " "
																						out_string += self.sp_model.decode(lowerCAmelCase__					) + token
																						snake_case_				=							True
																						snake_case_				=							[]
															else:
																						current_sub_tokens.append(lowerCAmelCase__					)
																						snake_case_				=							False
								out_string += self.sp_model.decode(lowerCAmelCase__					)
								return out_string.strip()




	def 							_UpperCamelCase			(       self       ,			a       ,			a = False       ,			a = None       ,			a = True       ,			**a       ,			)       ->    str:
								snake_case_				=							kwargs.pop('use_source_tokenizer'       ,			lowerCAmelCase__					)

								snake_case_				=							self.convert_ids_to_tokens(lowerCAmelCase__       ,			skip_special_tokens=lowerCAmelCase__					)

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								snake_case_				=							[]
								snake_case_				=							[]
								for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																						continue
															if token in self.added_tokens_encoder:
																						if current_sub_text:
																													sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__					)					)
																													snake_case_				=							[]
																						sub_texts.append(lowerCAmelCase__					)
															else:
																						current_sub_text.append(lowerCAmelCase__					)
								if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__					)					)

								# Mimic the behavior of the Rust tokenizer:
								# No space before [MASK] and [SEP]
								if spaces_between_special_tokens:
															snake_case_				=							re.sub(R' (\[(MASK|SEP)\])'       ,			R'\1'       ,			' '.join(lowerCAmelCase__					)					)
								else:
															snake_case_				=							''''''.join(lowerCAmelCase__					)

								snake_case_				=							(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
															snake_case_				=							self.clean_up_tokenization(lowerCAmelCase__					)
															return clean_text
								else:
															return text




	def 							_UpperCamelCase			(       self       ,			a       ,			a = None					)       ->    Tuple[str]:
								if not os.path.isdir(lowerCAmelCase__					):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
								snake_case_				=							os.path.join(
								    lowerCAmelCase__       ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

								if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase__					) and os.path.isfile(self.vocab_file					):
															copyfile(self.vocab_file       ,			lowerCAmelCase__					)
								elif not os.path.isfile(self.vocab_file					):
															with open(lowerCAmelCase__       ,			'wb'					) as fi:
																						snake_case_				=							self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase__					)

								return (out_vocab_file,)




	def 							_UpperCamelCase			(       self       ,			a       ,			a = None					)       ->    List[int]:
								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								snake_case_				=							[self.cls_token_id]
								snake_case_				=							[self.sep_token_id]
								return cls + token_ids_a + sep + token_ids_a + sep




	def 							_UpperCamelCase			(       self       ,			a       ,			a = None       ,			a = False					)       ->    List[int]:
								if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase__       ,			token_ids_a=lowerCAmelCase__       ,			already_has_special_tokens=lowerCAmelCase__					)

								if token_ids_a is None:
															return [1] + ([0] * len(lowerCAmelCase__					)) + [1]
								return [1] + ([0] * len(lowerCAmelCase__					)) + [1] + ([0] * len(lowerCAmelCase__					)) + [1]






	def 							_UpperCamelCase			(       self       ,			a       ,			a = None					)       ->    List[int]:
								snake_case_				=							[self.sep_token_id]
								snake_case_				=							[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]









import unittest

from transformers.testing_utils import require_bsa
from transformers.utils import is_bsa_available

from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin


if is_bsa_available():
						from transformers import MarkupLMFeatureExtractor
class        _lowercase      (			unittest.TestCase       ):






				'''simple docstring'''



				def __init__(       self			:Optional[Any]			,			lowerCAmelCase__			:Optional[Any]	)		->			str:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		parent



				def 		__magic_name__(       self			:List[Any]	)		->			Tuple:
										return {}




def        _UpperCamelCase							(      ):
						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		'''<HTML>

    <HEAD>
    <TITLE>sample document</TITLE>
    </HEAD>

    <BODY BGCOLOR="FFFFFF">
    <HR>
    <a href="http://google.com">Goog</a>
    <H1>This is one header</H1>
    <H2>This is a another Header</H2>
    <P>Travel from
        <P>
        <B>SFO to JFK</B>
        <BR>
        <B><I>on May 2, 2015 at 2:00 pm. For details go to confirm.com </I></B>
        <HR>
        <div style="color:#0000FF">
            <h3>Traveler <b> name </b> is
            <p> John Doe </p>
        </div>'''

						__SCREAMING_SNAKE_CASE						:     str					=		'''
    <!DOCTYPE html>
    <html>
    <body>

    <h1>My First Heading</h1>
    <p>My first paragraph.</p>

    </body>
    </html>
    '''

						return [html_string_a, html_string_a]
@require_bsa
class        _lowercase      (			A__						,			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		MarkupLMFeatureExtractor if is_bsa_available() else None


				def 		__magic_name__(       self			:int	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		MarkupLMFeatureExtractionTester(self	)


				@property
				def 		__magic_name__(       self			:Any	)		->			Optional[Any]:
										return self.feature_extract_tester.prepare_feat_extract_dict()



				def 		__magic_name__(       self			:Optional[int]	)		->			Any:
										# Initialize feature_extractor
										__SCREAMING_SNAKE_CASE						:     int					=		self.feature_extraction_class()

										# Test not batched input
										__SCREAMING_SNAKE_CASE						:     Tuple					=		get_html_strings()[0]
										__SCREAMING_SNAKE_CASE						:     Dict					=		feature_extractor(lowerCAmelCase__	)

										# fmt: off
										__SCREAMING_SNAKE_CASE						:     str					=		[['''sample document''', '''Goog''', '''This is one header''', '''This is a another Header''', '''Travel from''', '''SFO to JFK''', '''on May 2, 2015 at 2:00 pm. For details go to confirm.com''', '''Traveler''', '''name''', '''is''', '''John Doe''']]
										__SCREAMING_SNAKE_CASE						:     List[str]					=		[['''/html/head/title''', '''/html/body/a''', '''/html/body/h1''', '''/html/body/h2''', '''/html/body/p''', '''/html/body/p/p/b[1]''', '''/html/body/p/p/b[2]/i''', '''/html/body/p/p/div/h3''', '''/html/body/p/p/div/h3/b''', '''/html/body/p/p/div/h3''', '''/html/body/p/p/div/h3/p''']]
										# fmt: on

										self.assertEqual(encoding.nodes			,			lowerCAmelCase__	)
										self.assertEqual(encoding.xpaths			,			lowerCAmelCase__	)

										# Test batched
										__SCREAMING_SNAKE_CASE						:     Tuple					=		get_html_strings()
										__SCREAMING_SNAKE_CASE						:     Dict					=		feature_extractor(lowerCAmelCase__	)

										# fmt: off
										__SCREAMING_SNAKE_CASE						:     int					=		expected_nodes + [['''My First Heading''', '''My first paragraph.''']]
										__SCREAMING_SNAKE_CASE						:     str					=		expected_xpaths + [['''/html/body/h1''', '''/html/body/p''']]

										self.assertEqual(len(encoding.nodes	)			,			2	)
										self.assertEqual(len(encoding.xpaths	)			,			2	)

										self.assertEqual(encoding.nodes			,			lowerCAmelCase__	)
										self.assertEqual(encoding.xpaths			,			lowerCAmelCase__	)
