


import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			''''''
       A__											=			(
           None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
       )
       A__											=			None  # compression type in fsspec. ex: "gzip"
       A__											=			None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz





       def __init__(   self							:			List[str]					,      __lowerCamelCase							:			str = ""					,      __lowerCamelCase							:			Optional[str] = None					,      __lowerCamelCase							:			Optional[dict] = None					,      **__lowerCamelCase							:			Tuple       ):


             '''simple docstring'''



             super().__init__(self					,      **__SCREAMING_SNAKE_CASE       )
             # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
             lowerCamelCase__		:				int    =  fsspec.open(
                 __SCREAMING_SNAKE_CASE					,      mode="rb"					,      protocol=__SCREAMING_SNAKE_CASE					,      compression=self.compression					,      client_kwargs={
                     "requote_redirect_url": False,  # see https://github.com/huggingface/datasets/pull/5459
                     "trust_env": True,  # Enable reading proxy env variables.
                     **(target_options or {}).pop("client_kwargs"					,      {}       ),  # To avoid issues if it was already passed.
                 }					,      **(target_options or {})					,      )
             lowerCamelCase__		:				Optional[Any]    =  os.path.basename(self.file.path.split("::"       )[0]       )
             lowerCamelCase__		:				List[str]    =  (
                 self.compressed_name[: self.compressed_name.rindex("."       )]
                 if "." in self.compressed_name
                 else self.compressed_name
             )
             lowerCamelCase__		:				Tuple    =  None





       @classmethod
       def 						lowerCAmelCase     (   cls							:			Optional[Any]					,      __lowerCamelCase							:			int       ):


             '''simple docstring'''



             return super()._strip_protocol(__SCREAMING_SNAKE_CASE       ).lstrip("/"       )





       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             if self.dir_cache is None:
                   lowerCamelCase__		:				Any    =  {**self.file.fs.info(self.file.path       ), "name": self.uncompressed_name}
                   lowerCamelCase__		:				Optional[Any]    =  {f["name"]: f}





       def 						lowerCAmelCase     (   self							:			Any					,      __lowerCamelCase							:			str       ):


             '''simple docstring'''



             return self.file.open().read()





       def 						lowerCAmelCase     (   self							:			Tuple					,      __lowerCamelCase							:			str					,      __lowerCamelCase							:			str = "rb"					,      __lowerCamelCase							:			int=None					,      __lowerCamelCase							:			int=True					,      __lowerCamelCase							:			str=None					,      **__lowerCamelCase							:			int					,      ):


             '''simple docstring'''



             lowerCamelCase__		:				int    =  self._strip_protocol(__SCREAMING_SNAKE_CASE       )
             if mode != "rb":
                   raise ValueError(f"Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\'"       )
             return self.file.open()







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			'''bz2'''
       A__											=			'''bz2'''
       A__											=			'''.bz2'''







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			'''gzip'''
       A__											=			'''gzip'''
       A__											=			'''.gz'''







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			'''lz4'''
       A__											=			'''lz4'''
       A__											=			'''.lz4'''







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			'''xz'''
       A__											=			'''xz'''
       A__											=			'''.xz'''







class 			_lowercase (							__UpperCAmelCase):


       """simple docstring"""




       A__											=			'''zstd'''
       A__											=			'''zstd'''
       A__											=			'''.zst'''





       def __init__(   self							:			List[Any]					,      __lowerCamelCase							:			str					,      __lowerCamelCase							:			str = "rb"					,      __lowerCamelCase							:			Optional[str] = None					,      __lowerCamelCase							:			Optional[dict] = None					,      __lowerCamelCase							:			int = DEFAULT_BLOCK_SIZE					,      **__lowerCamelCase							:			str					,      ):


             '''simple docstring'''



             super().__init__(
                 fo=__SCREAMING_SNAKE_CASE					,      mode=__SCREAMING_SNAKE_CASE					,      target_protocol=__SCREAMING_SNAKE_CASE					,      target_options=__SCREAMING_SNAKE_CASE					,      block_size=__SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE					,      )
             # We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
             #
             # File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
             #     out.close = close
             # AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
             #
             # see https://github.com/intake/filesystem_spec/issues/725
             lowerCamelCase__		:				Tuple    =  self.file.__enter__







             class 			_lowercase :


                    """simple docstring"""




                    def __init__(   self							:			str					,      __lowerCamelCase							:			List[Any]       ):


                          '''simple docstring'''



                          lowerCamelCase__		:				Tuple    =  file_





                    def __enter__(   self							:			Union[str, Any]       ):


                          '''simple docstring'''



                          self._file.__enter__()
                          return self





                    def __exit__(   self							:			Optional[int]					,      *__lowerCamelCase							:			Union[str, Any]					,      **__lowerCamelCase							:			int       ):


                          '''simple docstring'''



                          self._file.__exit__(*__SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )





                    def __iter__(   self							:			Tuple       ):


                          '''simple docstring'''



                          return iter(self._file       )





                    def 						lowerCAmelCase     (   self							:			Dict       ):


                          '''simple docstring'''



                          return next(self._file       )





                    def __getattr__(   self							:			Optional[int]					,      __lowerCamelCase							:			Any       ):


                          '''simple docstring'''



                          return getattr(self._file					,      __SCREAMING_SNAKE_CASE       )

             def fixed_enter(*__lowerCamelCase							:			int					,      **__lowerCamelCase							:			Tuple       ):
                   return WrappedFile(_enter(*__SCREAMING_SNAKE_CASE					,      **__SCREAMING_SNAKE_CASE       )       )

             lowerCamelCase__		:				int    =  fixed_enter







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__snake_case						:str      		=     logging.get_logger(__name__)

__snake_case						:int      		=     {'''vocab_file''': '''vocab.txt'''}

__snake_case						:List[Any]      		=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__snake_case						:List[str]      		=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__snake_case						:Optional[int]      		=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    str											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase__  :    Optional[int]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    int											=						ConvBertTokenizer



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Dict=None     ,		__SCREAMING_SNAKE_CASE							:     Tuple=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     int="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[Any]="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     int="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		tokenizer_file=__SCREAMING_SNAKE_CASE     ,		do_lower_case=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,		strip_accents=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''     ,		__SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('''strip_accents'''     ,		__SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''     ,		__SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
               __a						       =				getattr(__SCREAMING_SNAKE_CASE     ,		normalizer_state.pop('''type'''))
               __a						       =				do_lower_case
               __a						       =				strip_accents
               __a						       =				tokenize_chinese_chars
               __a						       =				normalizer_class(**__SCREAMING_SNAKE_CASE)

           __a						       =				do_lower_case



       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=None):


           '''simple docstring'''

           __a						       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

           return output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           __a						       =				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,		name=__SCREAMING_SNAKE_CASE)
           return tuple(__SCREAMING_SNAKE_CASE)





