


'''simple docstring'''







import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


A__:    Optional[Any]					    =       get_tests_dir('''fixtures/test_sentencepiece.model''')







@require_sentencepiece
@require_tokenizers
class A__       (						_snake_case ,     unittest.TestCase      ):
      __UpperCamelCase       :  int	       =							XLNetTokenizer
      __UpperCamelCase       :  int	       =							XLNetTokenizerFast
      __UpperCamelCase       :  Union[str, Any]	       =							True
      __UpperCamelCase       :  List[Any]	       =							True






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					List[Any]:




             '''simple docstring'''
             super().setUp()

             # We have a SentencePiece fixture for testing
             _a    : Optional[Any]     					=XLNetTokenizer(_UpperCamelCase     ,						keep_accents=_UpperCamelCase	)
             tokenizer.sanitize_special_tokens()
             tokenizer.save_pretrained(self.tmpdirname	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Dict     					='<s>'
             _a    : str     					=1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase	)     ,						_UpperCamelCase	)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase	)     ,						_UpperCamelCase	)






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : str     					=list(self.get_tokenizer().get_vocab().keys()	)

             self.assertEqual(vocab_keys[0]     ,						"""<unk>"""	)
             self.assertEqual(vocab_keys[1]     ,						"""<s>"""	)
             self.assertEqual(vocab_keys[-1]     ,						"""<eod>"""	)
             self.assertEqual(len(_UpperCamelCase	)     ,						1_0_0_6	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Dict:




             '''simple docstring'''
             self.assertEqual(self.get_tokenizer().vocab_size     ,						1_0_0_0	)






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Optional[Any]     					=XLNetTokenizer(_UpperCamelCase     ,						keep_accents=_UpperCamelCase	)

             _a    : Tuple     					=tokenizer.tokenize("""This is a test"""	)
             self.assertListEqual(_UpperCamelCase     ,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

             self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase	)     ,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]	)

             _a    : int     					=tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
             self.assertListEqual(
                 _UpperCamelCase     ,						[
                     SPIECE_UNDERLINE + """I""",
                     SPIECE_UNDERLINE + """was""",
                     SPIECE_UNDERLINE + """b""",
                     """or""",
                     """n""",
                     SPIECE_UNDERLINE + """in""",
                     SPIECE_UNDERLINE + """""",
                     """9""",
                     """2""",
                     """0""",
                     """0""",
                     """0""",
                     """,""",
                     SPIECE_UNDERLINE + """and""",
                     SPIECE_UNDERLINE + """this""",
                     SPIECE_UNDERLINE + """is""",
                     SPIECE_UNDERLINE + """f""",
                     """al""",
                     """s""",
                     """é""",
                     """.""",
                 ]     ,						)
             _a    : Optional[int]     					=tokenizer.convert_tokens_to_ids(_UpperCamelCase	)
             self.assertListEqual(_UpperCamelCase     ,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]	)

             _a    : Union[str, Any]     					=tokenizer.convert_ids_to_tokens(_UpperCamelCase	)
             self.assertListEqual(
                 _UpperCamelCase     ,						[
                     SPIECE_UNDERLINE + """I""",
                     SPIECE_UNDERLINE + """was""",
                     SPIECE_UNDERLINE + """b""",
                     """or""",
                     """n""",
                     SPIECE_UNDERLINE + """in""",
                     SPIECE_UNDERLINE + """""",
                     """<unk>""",
                     """2""",
                     """0""",
                     """0""",
                     """0""",
                     """,""",
                     SPIECE_UNDERLINE + """and""",
                     SPIECE_UNDERLINE + """this""",
                     SPIECE_UNDERLINE + """is""",
                     SPIECE_UNDERLINE + """f""",
                     """al""",
                     """s""",
                     """<unk>""",
                     """.""",
                 ]     ,						)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Tuple:




             '''simple docstring'''
             _a    : List[Any]     					=XLNetTokenizer(_UpperCamelCase     ,						do_lower_case=_UpperCamelCase	)
             _a    : List[str]     					=tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
             self.assertListEqual(
                 _UpperCamelCase     ,						[
                     SPIECE_UNDERLINE + """""",
                     """i""",
                     SPIECE_UNDERLINE + """was""",
                     SPIECE_UNDERLINE + """b""",
                     """or""",
                     """n""",
                     SPIECE_UNDERLINE + """in""",
                     SPIECE_UNDERLINE + """""",
                     """9""",
                     """2""",
                     """0""",
                     """0""",
                     """0""",
                     """,""",
                     SPIECE_UNDERLINE + """and""",
                     SPIECE_UNDERLINE + """this""",
                     SPIECE_UNDERLINE + """is""",
                     SPIECE_UNDERLINE + """f""",
                     """al""",
                     """se""",
                     """.""",
                 ]     ,						)
             self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)     ,						["""▁he""", """ll""", """o"""]	)






      def __UpperCAmelCase       ( self			:str	)       ->					Any:




             '''simple docstring'''
             _a    : Dict     					=XLNetTokenizer(_UpperCamelCase     ,						do_lower_case=_UpperCamelCase	)
             _a    : List[Any]     					=tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
             self.assertListEqual(
                 _UpperCamelCase     ,						[
                     SPIECE_UNDERLINE + """I""",
                     SPIECE_UNDERLINE + """was""",
                     SPIECE_UNDERLINE + """b""",
                     """or""",
                     """n""",
                     SPIECE_UNDERLINE + """in""",
                     SPIECE_UNDERLINE + """""",
                     """9""",
                     """2""",
                     """0""",
                     """0""",
                     """0""",
                     """,""",
                     SPIECE_UNDERLINE + """and""",
                     SPIECE_UNDERLINE + """this""",
                     SPIECE_UNDERLINE + """is""",
                     SPIECE_UNDERLINE + """f""",
                     """al""",
                     """se""",
                     """.""",
                 ]     ,						)






      @slow
      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Dict:




             '''simple docstring'''
             _a    : str     					=XLNetTokenizer.from_pretrained("""xlnet-base-cased"""	)

             _a    : Union[str, Any]     					=tokenizer.encode("""sequence builders"""     ,						add_special_tokens=_UpperCamelCase	)
             _a    : List[Any]     					=tokenizer.encode("""multi-sequence build"""     ,						add_special_tokens=_UpperCamelCase	)

             _a    : Optional[int]     					=tokenizer.build_inputs_with_special_tokens(_UpperCamelCase	)
             _a    : Tuple     					=tokenizer.build_inputs_with_special_tokens(_UpperCamelCase     ,						_UpperCamelCase	)

             assert encoded_sentence == text + [4, 3]
             assert encoded_pair == text + [4] + text_a + [4, 3]






      @slow
      def __UpperCAmelCase       ( self			:Dict	)       ->					str:




             '''simple docstring'''
             # fmt: off
             _a    : List[Any]     					={'input_ids': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=_UpperCamelCase     ,						model_name="""xlnet-base-cased"""     ,						revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""     ,						)










import importlib
import shutil
import threading
import warnings
from typing import List

import fsspec
import fsspec.asyn

from . import compression
from .hffilesystem import HfFileSystem


__UpperCAmelCase								=						importlib.util.find_spec('s3fs') is not None

if _has_safs:
		from .safilesystem import SaFileSystem  # noqa: F401

__UpperCAmelCase								=						[
    compression.BzaFileSystem,
    compression.GzipFileSystem,
    compression.LzaFileSystem,
    compression.XzFileSystem,
    compression.ZstdFileSystem,
]

# Register custom filesystems
for fs_class in COMPRESSION_FILESYSTEMS + [HfFileSystem]:
		if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
				warnings.warn(F'A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.')
		fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)




def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					if "://" in dataset_path:
										UpperCAmelCase_    :      int            =						dataset_path.split('://'  )[1]
					return dataset_path




def      lowercase__      (     __snake_case    :							fsspec.AbstractFileSystem  ):


					'''simple docstring'''


					if fs is not None and fs.protocol != "file":
										return True
					else:
										return False




def      lowercase__      (     __snake_case    :							fsspec.AbstractFileSystem     ,       __snake_case    :							str     ,       __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						not is_remote_filesystem(__snake_case  )
					if is_local:
										# LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
										shutil.move(fs._strip_protocol(__snake_case  )     ,       fs._strip_protocol(__snake_case  )  )
					else:
										fs.mv(__snake_case     ,       __snake_case     ,       recursive=__snake_case  )






def      lowercase__      (     ):


					'''simple docstring'''


					if hasattr(fsspec.asyn     ,       'reset_lock'  ):
										# for future fsspec>2022.05.0
										fsspec.asyn.reset_lock()
					else:
										UpperCAmelCase_    :      Optional[Any]            =						None
										UpperCAmelCase_    :      Union[str, Any]            =						None
										UpperCAmelCase_    :      int            =						threading.Lock()






