

import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_       	=	get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
@require_tokenizers
class 	snake_case_   (		__A	,       unittest.TestCase					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Optional[int]									=				ReformerTokenizer
			SCREAMING_SNAKE_CASE     :   int									=				ReformerTokenizerFast
			SCREAMING_SNAKE_CASE     :   int									=				True
			SCREAMING_SNAKE_CASE     :   Optional[Any]									=				False
			SCREAMING_SNAKE_CASE     :   str									=				True







			def 				snake_case__(     self :				Dict							)      ->List[str]:
									super().setUp()

									snake_case_   =	ReformerTokenizer(_UpperCamelCase							,   keep_accents=_UpperCamelCase							)
									tokenizer.save_pretrained(self.tmpdirname							)







			def 				snake_case__(     self :				Optional[Any]							)      ->int:
									snake_case_   =	'''<s>'''
									snake_case_   =	1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase							)							,   _UpperCamelCase							)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase							)							,   _UpperCamelCase							)







			def 				snake_case__(     self :				Optional[Any]							)      ->Dict:
									snake_case_   =	list(self.get_tokenizer().get_vocab().keys()							)

									self.assertEqual(vocab_keys[0]							,   '''<unk>'''							)
									self.assertEqual(vocab_keys[1]							,   '''<s>'''							)
									self.assertEqual(vocab_keys[-1]							,   '''j'''							)
									self.assertEqual(len(_UpperCamelCase							)							,   1_0_0_0							)







			def 				snake_case__(     self :				Union[str, Any]							)      ->Dict:
									self.assertEqual(self.get_tokenizer().vocab_size							,   1_0_0_0							)







			def 				snake_case__(     self :				List[Any]							)      ->str:
									if not self.test_rust_tokenizer:
															return

									snake_case_   =	self.get_tokenizer()
									snake_case_   =	self.get_rust_tokenizer()

									snake_case_   =	'''I was born in 92000, and this is falsé.'''

									snake_case_   =	tokenizer.tokenize(_UpperCamelCase							)
									snake_case_   =	rust_tokenizer.tokenize(_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)

									snake_case_   =	tokenizer.encode(_UpperCamelCase							,   add_special_tokens=_UpperCamelCase							)
									snake_case_   =	rust_tokenizer.encode(_UpperCamelCase							,   add_special_tokens=_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)

									snake_case_   =	self.get_rust_tokenizer()
									snake_case_   =	tokenizer.encode(_UpperCamelCase							)
									snake_case_   =	rust_tokenizer.encode(_UpperCamelCase							)
									self.assertListEqual(_UpperCamelCase							,   _UpperCamelCase							)







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				Tuple=1_5							)      ->Tuple:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																					snake_case_   =	self.rust_tokenizer_class.from_pretrained(_UpperCamelCase							,   **_UpperCamelCase							)

																					# Simple input
																					snake_case_   =	'''This is a simple input'''
																					snake_case_   =	['''This is a simple input 1''', '''This is a simple input 2''']
																					snake_case_   =	('''This is a simple input''', '''This is a pair''')
																					snake_case_   =	[
																					    ('''This is a simple input 1''', '''This is a simple input 2'''),
																					    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																					]

																					# Simple input tests
																					self.assertRaises(_UpperCamelCase							,   tokenizer_r.encode							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							)

																					# Simple input
																					self.assertRaises(_UpperCamelCase							,   tokenizer_r.encode_plus							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							)

																					# Simple input
																					self.assertRaises(
																					    _UpperCamelCase							,   tokenizer_r.batch_encode_plus							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							,   )

																					# Pair input
																					self.assertRaises(_UpperCamelCase							,   tokenizer_r.encode							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							)

																					# Pair input
																					self.assertRaises(_UpperCamelCase							,   tokenizer_r.encode_plus							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							)

																					# Pair input
																					self.assertRaises(
																					    _UpperCamelCase							,   tokenizer_r.batch_encode_plus							,   _UpperCamelCase							,   max_length=_UpperCamelCase							,   padding='''max_length'''							,   )







			def 				snake_case__(     self :				Dict							)      ->int:
									pass







			def 				snake_case__(     self :				Dict							)      ->str:
									snake_case_   =	ReformerTokenizer(_UpperCamelCase							,   keep_accents=_UpperCamelCase							)

									snake_case_   =	tokenizer.tokenize('''This is a test'''							)
									self.assertListEqual(_UpperCamelCase							,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_UpperCamelCase							)							,   [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]							,   )

									snake_case_   =	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
									self.assertListEqual(
									    _UpperCamelCase							,   [
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''9''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''é''',
									        '''.''',
									    ]							,   )
									snake_case_   =	tokenizer.convert_tokens_to_ids(_UpperCamelCase							)
									self.assertListEqual(
									    _UpperCamelCase							,   [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]							,   )

									snake_case_   =	tokenizer.convert_ids_to_tokens(_UpperCamelCase							)
									self.assertListEqual(
									    _UpperCamelCase							,   [
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''<unk>''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''<unk>''',
									        '''.''',
									    ]							,   )







			@cached_property
			def 				snake_case__(     self :				Optional[int]							)      ->Tuple:
									return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''							)







			@slow
			def 				snake_case__(     self :				Optional[Any]							)      ->int:
									snake_case_   =	'''Hello World!'''
									snake_case_   =	[1_2_6, 3_2, 2_6_2, 1_5_2, 3_8, 7_2, 2_8_7]

									self.assertListEqual(_UpperCamelCase							,   self.big_tokenizer.encode(_UpperCamelCase							)							)







			@slow
			def 				snake_case__(     self :				str							)      ->str:
									snake_case_   =	(
									    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
									    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
									)
									snake_case_   =	[
									    1_0_8,
									    2_6_5,
									    2_4,
									    1_1_1,
									    4,
									    2_5_8,
									    1_5_6,
									    3_5,
									    2_8,
									    2_7_5,
									    3,
									    2_5_9,
									    2_9_7,
									    2_6_0,
									    8_4,
									    4,
									    3_5,
									    1_1_0,
									    4_4,
									    8,
									    2_5_9,
									    9_1,
									    2_6_8,
									    2_1,
									    1_1,
									    2_0_9,
									    2_7_4,
									    1_0_9,
									    2_6_6,
									    2_7_7,
									    1_1_7,
									    8_6,
									    9_3,
									    3_1_5,
									    2_5_8,
									    2_7_8,
									    2_5_8,
									    2_7_7,
									    2_5_8,
									    0,
									    2_5_8,
									    2_8_8,
									    2_5_8,
									    3_1_9,
									    2_5_8,
									    0,
									    2_5_8,
									    0,
									    2_5_8,
									    0,
									    2_5_8,
									    0,
									    2_5_8,
									    2_8_7,
									    2_5_8,
									    3_1_5,
									    2_5_8,
									    2_8_9,
									    2_5_8,
									    2_7_8,
									    9_9,
									    2_6_9,
									    2_6_6,
									    2_6_2,
									    8,
									    2_5_9,
									    2_4_1,
									    4,
									    2_1_7,
									    2_3_0,
									    2_6_8,
									    2_6_6,
									    5_5,
									    1_6_8,
									    1_0_6,
									    7_5,
									    1_9_3,
									    2_6_6,
									    2_2_3,
									    2_7,
									    4_9,
									    2_6,
									    2_8_2,
									    2_5,
									    2_6_4,
									    2_9_9,
									    1_9,
									    2_6,
									    0,
									    2_5_8,
									    2_7_7,
									    1_1_7,
									    8_6,
									    9_3,
									    1_7_6,
									    1_8_3,
									    2_7_0,
									    1_1,
									    2_6_2,
									    4_2,
									    6_1,
									    2_6_5,
									]

									self.assertListEqual(_UpperCamelCase							,   self.big_tokenizer.encode(_UpperCamelCase							)							)







			@require_torch
			@slow
			def 				snake_case__(     self :				List[str]							)      ->List[str]:
									import torch

									from transformers import ReformerConfig, ReformerModel

									# Build sequence
									snake_case_   =	list(self.big_tokenizer.get_vocab().keys()							)[:1_0]
									snake_case_   =	''' '''.join(_UpperCamelCase							)
									snake_case_   =	self.big_tokenizer.encode_plus(_UpperCamelCase							,   return_tensors='''pt'''							)
									snake_case_   =	self.big_tokenizer.batch_encode_plus([sequence, sequence]							,   return_tensors='''pt'''							)

									snake_case_   =	ReformerConfig()
									# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
									snake_case_   =	encoded_sequence['''input_ids'''].shape
									snake_case_   =	ReformerModel(_UpperCamelCase							)

									# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
									assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

									with torch.no_grad():
															model(**_UpperCamelCase							)
															model(**_UpperCamelCase							)







			@slow
			def 				snake_case__(     self :				List[Any]							)      ->Dict:
									# fmt: off
									snake_case_   =	{'''input_ids''': [[1_0_8, 2_6_5, 2_4, 1_1_1, 4, 2_5_8, 1_5_6, 7, 5_1, 2_7_9, 5_8, 7, 7_6, 2_5, 6_9, 2_7_8], [1_4_0, 2_4_3, 2_6_4, 1_3_4, 1_7, 2_6_7, 7_7, 2_6_3, 2_2, 2_6_2, 2_9_7, 2_5_8, 3_0_4, 1_7_7, 2_7_9, 2_6_6, 1_4, 8_9, 1_3, 3_5, 2_6_1, 2_9_9, 2_7_2, 1_3_7, 2_7_5, 2_7_8]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
									# fmt: on

									# This tokenizer does not know some characters like ")".
									# That is the reason why we use very simple texts here.
									# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
									snake_case_   =	[
									    '''This is a very simple sentence.''',
									    '''The quick brown fox jumps over the lazy dog.''',
									]

									self.tokenizer_integration_test_util(
									    expected_encoding=_UpperCamelCase							,   model_name='''google/reformer-crime-and-punishment'''							,   revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''							,   padding=_UpperCamelCase							,   sequences=_UpperCamelCase							,   )

# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


SCREAMING_SNAKE_CASE							:Union[str, Any]							 =				get_logger(__name__)





class 							UpperCAmelCase      :




	'''simple docstring'''





	snake_case_					    =     "dummy_data"
	snake_case_					    =     "datasets"
	snake_case_					    =     False






	def __init__(       self					: Optional[int]							,A					: str							,A					: str							,A					: Union[Version, str]							,A					: Optional[str] = None							,A					: bool = False							,A					: bool = True							,A					: Optional[List[Callable]] = None							,):
					__A   			= 0
					__A   			= dataset_name
					__A   			= cache_dir
					__A   			= use_local_dummy_data
					__A   			= config
					# download_callbacks take a single url as input
					__A   			= download_callbacks or []
					# if False, it doesn't load existing files and it returns the paths of the dummy files relative
					# to the dummy_data zip file root
					__A   			= load_existing_dummy_data

					# TODO(PVP, QL) might need to make this more general
					__A   			= str(A )
					# to be downloaded
					__A   			= None
					__A   			= None






	@property
	def 				UpperCamelCase_  (       self					: Union[str, Any] ):
					if self._dummy_file is None:
									__A   			= self.download_dummy_data()
					return self._dummy_file






	@property
	def 				UpperCamelCase_  (       self					: Optional[Any] ):
					if self.config is not None:
									# structure is dummy / config_name / version_name
									return os.path.join("dummy"							,self.config.name							,self.version_name )
					# structure is dummy / version_name
					return os.path.join("dummy"							,self.version_name )






	@property
	def 				UpperCamelCase_  (       self					: List[Any] ):
					return os.path.join(self.dummy_data_folder							,"dummy_data.zip" )






	def 				UpperCamelCase_  (       self					: Tuple ):
					__A   			= (
					    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
					)

					__A   			= cached_path(
					    A							,cache_dir=self.cache_dir							,extract_compressed_file=A							,force_extract=A )

					return os.path.join(A							,self.dummy_file_name )






	@property
	def 				UpperCamelCase_  (       self					: str ):
					return os.path.join(self.datasets_scripts_dir							,self.dataset_name							,self.dummy_zip_file )






	@property
	def 				UpperCamelCase_  (       self					: Any ):
					if self._bucket_url is None:
									__A   			= hf_github_url(self.dataset_name							,self.dummy_zip_file.replace(os.sep							,"/" ) )
					return self._bucket_url






	@property
	def 				UpperCamelCase_  (       self					: Tuple ):
					# return full path if its a dir
					if os.path.isdir(self.dummy_file ):
									return self.dummy_file
					# else cut off path to file -> example `xsum`.
					return "/".join(self.dummy_file.replace(os.sep							,"/" ).split("/" )[:-1] )






	def 				UpperCamelCase_  (       self					: List[str]							,A					: List[Any]							,*A					: Dict ):
					if self.load_existing_dummy_data:
									# dummy data is downloaded and tested
									__A   			= self.dummy_file
					else:
									# dummy data cannot be downloaded and only the path to dummy file is returned
									__A   			= self.dummy_file_name

					# special case when data_url is a dict
					if isinstance(A							,A ):
									return self.create_dummy_data_dict(A							,A )
					elif isinstance(A							,(list, tuple) ):
									return self.create_dummy_data_list(A							,A )
					else:
									return self.create_dummy_data_single(A							,A )






	def 				UpperCamelCase_  (       self					: str							,A					: List[Any]							,*A					: List[Any] ):
					return self.download_and_extract(A )






	def 				UpperCamelCase_  (       self					: List[str]							,A					: List[str]							,A					: Tuple ):
					return self.download_and_extract(A )






	def 				UpperCamelCase_  (       self					: Any							,A					: Any							,*A					: Optional[Any]							,**A					: List[str] ):
					return path






	def 				UpperCamelCase_  (       self					: str ):
					return {}






	def 				UpperCamelCase_  (       self					: int							,A					: int							,A					: Tuple ):
					__A   			= {}
					for key, single_urls in data_url.items():
									for download_callback in self.download_callbacks:
													if isinstance(A							,A ):
																	for single_url in single_urls:
																					download_callback(A )
													else:
																	__A   			= single_urls
																	download_callback(A )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
									if isinstance(A							,A ):
													__A   			= [os.path.join(A							,urllib.parse.quote_plus(Path(A ).name ) ) for x in single_urls]
									else:
													__A   			= single_urls
													__A   			= os.path.join(A							,urllib.parse.quote_plus(Path(A ).name ) )
									__A   			= value

					# make sure that values are unique
					if all(isinstance(A							,A ) for i in dummy_data_dict.values() ) and len(set(dummy_data_dict.values() ) ) < len(
					    dummy_data_dict.values() ):
									# append key to value to make its name unique
									__A   			= {key: value + key for key, value in dummy_data_dict.items()}

					return dummy_data_dict






	def 				UpperCamelCase_  (       self					: Union[str, Any]							,A					: str							,A					: str ):
					__A   			= []
					# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
					__A   			= all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}"							,A ) ) for url in data_url )
					__A   			= all(
					    url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed" ) for url in data_url )
					if data_url and (is_tf_records or is_pubmed_records):
									__A   			= [data_url[0]] * len(A )
					for single_url in data_url:
									for download_callback in self.download_callbacks:
													download_callback(A )
									# we force the name of each key to be the last file / folder name of the url path
									# if the url has arguments, we need to encode them with urllib.parse.quote_plus
									__A   			= os.path.join(A							,urllib.parse.quote_plus(single_url.split("/" )[-1] ) )
									dummy_data_list.append(A )
					return dummy_data_list






	def 				UpperCamelCase_  (       self					: str							,A					: List[Any]							,A					: Optional[Any] ):
					for download_callback in self.download_callbacks:
									download_callback(A )
					# we force the name of each key to be the last file / folder name of the url path
					# if the url has arguments, we need to encode them with urllib.parse.quote_plus
					__A   			= os.path.join(A							,urllib.parse.quote_plus(data_url.split("/" )[-1] ) )
					if os.path.exists(A ) or not self.load_existing_dummy_data:
									return value
					else:
									# Backward compatibility, maybe deprecate at one point.
									# For many datasets with single url calls to dl_manager.download_and_extract,
									# the dummy_data.zip file is actually the zipped downloaded file
									# while now we expected the dummy_data.zip file to be a directory containing
									# the downloaded file.
									return path_to_dummy_data






	def 				UpperCamelCase_  (       self					: int ):
					pass






	def 				UpperCamelCase_  (       self					: Dict ):
					pass






	def 				UpperCamelCase_  (       self					: Optional[Any]							,A					: List[Any] ):
					def _iter_archive_members(A					: Optional[Any] ):
									# this preserves the order of the members inside the ZIP archive
									__A   			= Path(self.dummy_file ).parent
									__A   			= path.relative_to(A )
									with ZipFile(self.local_path_to_dummy_data ) as zip_file:
													__A   			= zip_file.namelist()
									for member in members:
													if member.startswith(relative_path.as_posix() ):
																	yield dummy_parent_path.joinpath(A )

					__A   			= Path(A )
					__A   			= _iter_archive_members(A ) if self.use_local_dummy_data else path.rglob("*" )
					for file_path in file_paths:
									if file_path.is_file() and not file_path.name.startswith((".", "__") ):
													yield file_path.relative_to(A ).as_posix(), file_path.open("rb" )







	def 				UpperCamelCase_  (       self					: List[Any]							,A					: Any ):
					if not isinstance(A							,A ):
									__A   			= [paths]
					for path in paths:
									if os.path.isfile(A ):
													if os.path.basename(A ).startswith((".", "__") ):
																	return
													yield path
									else:
													for dirpath, dirnames, filenames in os.walk(A ):
																	if os.path.basename(A ).startswith((".", "__") ):
																					continue
																	dirnames.sort()
																	for filename in sorted(A ):
																					if filename.startswith((".", "__") ):
																									continue
																					yield os.path.join(A							,A )
