



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__snake_case						:str      		=     logging.get_logger(__name__)

__snake_case						:int      		=     {'''vocab_file''': '''vocab.txt'''}

__snake_case						:List[Any]      		=     {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__snake_case						:List[str]      		=     {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__snake_case						:Optional[int]      		=     {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class     _A       (	__UpperCAmelCase    ):
       UpperCamelCase__  :    Optional[int]											=						VOCAB_FILES_NAMES
       UpperCamelCase__  :    str											=						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :    Optional[Any]											=						PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase__  :    Optional[int]											=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :    int											=						ConvBertTokenizer



       def __init__(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]=None     ,		__SCREAMING_SNAKE_CASE							:     Dict=None     ,		__SCREAMING_SNAKE_CASE							:     Tuple=True     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[UNK]"     ,		__SCREAMING_SNAKE_CASE							:     int="[SEP]"     ,		__SCREAMING_SNAKE_CASE							:     List[Any]="[PAD]"     ,		__SCREAMING_SNAKE_CASE							:     int="[CLS]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]="[MASK]"     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]=True     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=None     ,		**__SCREAMING_SNAKE_CASE							:     Dict     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		tokenizer_file=__SCREAMING_SNAKE_CASE     ,		do_lower_case=__SCREAMING_SNAKE_CASE     ,		unk_token=__SCREAMING_SNAKE_CASE     ,		sep_token=__SCREAMING_SNAKE_CASE     ,		pad_token=__SCREAMING_SNAKE_CASE     ,		cls_token=__SCREAMING_SNAKE_CASE     ,		mask_token=__SCREAMING_SNAKE_CASE     ,		tokenize_chinese_chars=__SCREAMING_SNAKE_CASE     ,		strip_accents=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)

           __a						       =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
           if (
               normalizer_state.get('''lowercase'''     ,		__SCREAMING_SNAKE_CASE) != do_lower_case
               or normalizer_state.get('''strip_accents'''     ,		__SCREAMING_SNAKE_CASE) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''     ,		__SCREAMING_SNAKE_CASE) != tokenize_chinese_chars
           ):
               __a						       =				getattr(__SCREAMING_SNAKE_CASE     ,		normalizer_state.pop('''type'''))
               __a						       =				do_lower_case
               __a						       =				strip_accents
               __a						       =				tokenize_chinese_chars
               __a						       =				normalizer_class(**__SCREAMING_SNAKE_CASE)

           __a						       =				do_lower_case



       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=None):


           '''simple docstring'''

           __a						       =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

           return output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     List[int]     ,		__SCREAMING_SNAKE_CASE							:     Optional[List[int]] = None):


           '''simple docstring'''

           __a						       =				[self.sep_token_id]
           __a						       =				[self.cls_token_id]
           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





       def 		_lowerCamelCase			(		self							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[str] = None):


           '''simple docstring'''

           __a						       =				self._tokenizer.model.save(__SCREAMING_SNAKE_CASE     ,		name=__SCREAMING_SNAKE_CASE)
           return tuple(__SCREAMING_SNAKE_CASE)










import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


__snake_case						:List[str]      		=     '''\
    Text data.
    Second line of data.'''

__snake_case						:Optional[Any]      		=     '''file'''





@pytest.fixture(scope='''session'''				)
def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				tmp_path_factory.mktemp('''data'''				) / (FILE_PATH + '''.zstd''')
    __a						       =				bytes(_UpperCAmelCase   , '''utf-8'''				)
    with zstd.open(_UpperCAmelCase   , '''wb'''				) as f:
        f.write(_UpperCAmelCase				)
    return path





@pytest.fixture
def      __snake_case	(			_UpperCAmelCase				):
    with open(os.path.join(tmpfs.local_root_dir   , _UpperCAmelCase				)   , '''w'''				) as f:
        f.write(_UpperCAmelCase				)
    return FILE_PATH





@pytest.mark.parametrize('''compression_format'''   , ['''gzip''', '''xz''', '''zstd''']				)
def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase				):
    __a						       =				{'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
    __a						       =				input_paths[compression_format]
    __a						       =				tmp_path / '''cache'''
    __a						       =				DownloadConfig(cache_dir=_UpperCAmelCase   , extract_compressed_file=_UpperCAmelCase				)
    __a						       =				cached_path(_UpperCAmelCase   , download_config=_UpperCAmelCase				)
    with open(_UpperCAmelCase				) as f:
        __a						       =				f.read()
    with open(_UpperCAmelCase				) as f:
        __a						       =				f.read()
    assert extracted_file_content == expected_file_content





@pytest.mark.parametrize('''default_extracted'''   , [True, False]				)
@pytest.mark.parametrize('''default_cache_dir'''   , [True, False]				)
def      __snake_case	(			_UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase   , _UpperCAmelCase				):
    __a						       =				'''custom_cache'''
    __a						       =				'''custom_extracted_dir'''
    __a						       =				tmp_path / '''custom_extracted_path'''
    if default_extracted:
        __a						       =				('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
    else:
        monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''   , _UpperCAmelCase				)
        monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''   , str(_UpperCAmelCase				)				)
        __a						       =				custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

    __a						       =				xz_file
    __a						       =				(
        DownloadConfig(extract_compressed_file=_UpperCAmelCase				)
        if default_cache_dir
        else DownloadConfig(cache_dir=tmp_path / custom_cache_dir   , extract_compressed_file=_UpperCAmelCase				)
    )
    __a						       =				cached_path(_UpperCAmelCase   , download_config=_UpperCAmelCase				)
    assert Path(_UpperCAmelCase				).parent.parts[-2:] == expected





def      __snake_case	(			_UpperCAmelCase				):
    # absolute path
    __a						       =				str(Path(_UpperCAmelCase				).resolve()				)
    assert cached_path(_UpperCAmelCase				) == text_file
    # relative path
    __a						       =				str(Path(_UpperCAmelCase				).resolve().relative_to(Path(os.getcwd()				)				)				)
    assert cached_path(_UpperCAmelCase				) == text_file





def      __snake_case	(			_UpperCAmelCase				):
    # absolute path
    __a						       =				str(tmp_path.resolve() / '''__missing_file__.txt'''				)
    with pytest.raises(_UpperCAmelCase				):
        cached_path(_UpperCAmelCase				)
    # relative path
    __a						       =				'''./__missing_file__.txt'''
    with pytest.raises(_UpperCAmelCase				):
        cached_path(_UpperCAmelCase				)





def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				get_from_cache(f'tmp://{tmpfs_file}'				)
    with open(_UpperCAmelCase				) as f:
        __a						       =				f.read()
    assert output_file_content == FILE_CONTENT





@patch('''datasets.config.HF_DATASETS_OFFLINE'''   , _UpperCAmelCase				)
def      __snake_case	(			):
    with pytest.raises(_UpperCAmelCase				):
        cached_path('''https://huggingface.co'''				)





@patch('''datasets.config.HF_DATASETS_OFFLINE'''   , _UpperCAmelCase				)
def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				tmp_path_factory.mktemp('''data'''				) / '''file.html'''
    with pytest.raises(_UpperCAmelCase				):
        http_get('''https://huggingface.co'''   , temp_file=_UpperCAmelCase				)
    with pytest.raises(_UpperCAmelCase				):
        http_head('''https://huggingface.co'''				)





@patch('''datasets.config.HF_DATASETS_OFFLINE'''   , _UpperCAmelCase				)
def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				tmp_path_factory.mktemp('''data'''				) / '''file.html'''
    with pytest.raises(_UpperCAmelCase				):
        ftp_get('''ftp://huggingface.co'''   , temp_file=_UpperCAmelCase				)
    with pytest.raises(_UpperCAmelCase				):
        ftp_head('''ftp://huggingface.co'''				)





@patch('''datasets.config.HF_DATASETS_OFFLINE'''   , _UpperCAmelCase				)
def      __snake_case	(			_UpperCAmelCase				):
    __a						       =				tmp_path_factory.mktemp('''data'''				) / '''file.html'''
    with pytest.raises(_UpperCAmelCase				):
        fsspec_get('''s3://huggingface.co'''   , temp_file=_UpperCAmelCase				)
    with pytest.raises(_UpperCAmelCase				):
        fsspec_head('''s3://huggingface.co'''				)





