


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__lowerCAmelCase	: Dict  				={'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	: List[str]  				={
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase	: List[str]  				={
    'unc-nlp/lxmert-base-uncased': 5_1_2,
}

__lowerCAmelCase	: Any  				={
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}
class        _lowercase      (			A__       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			List[str]						=		VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__				:			Optional[int]						=		PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__				:			Tuple						=		PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE__				:			List[Any]						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__				:			List[str]						=		LxmertTokenizer


				def __init__(       self			:int			,			lowerCAmelCase__			:Dict=None			,			lowerCAmelCase__			:Optional[Any]=None			,			lowerCAmelCase__			:Any=True			,			lowerCAmelCase__			:str="[UNK]"			,			lowerCAmelCase__			:Optional[int]="[SEP]"			,			lowerCAmelCase__			:str="[PAD]"			,			lowerCAmelCase__			:Optional[Any]="[CLS]"			,			lowerCAmelCase__			:List[Any]="[MASK]"			,			lowerCAmelCase__			:Union[str, Any]=True			,			lowerCAmelCase__			:List[Any]=None			,			**lowerCAmelCase__			:str			,			)		->			Dict:
										super().__init__(
										    lowerCAmelCase__			,			tokenizer_file=lowerCAmelCase__			,			do_lower_case=lowerCAmelCase__			,			unk_token=lowerCAmelCase__			,			sep_token=lowerCAmelCase__			,			pad_token=lowerCAmelCase__			,			cls_token=lowerCAmelCase__			,			mask_token=lowerCAmelCase__			,			tokenize_chinese_chars=lowerCAmelCase__			,			strip_accents=lowerCAmelCase__			,			**lowerCAmelCase__			,			)

										__SCREAMING_SNAKE_CASE						:     Any					=		json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
										if (
										    normalizer_state.get('''lowercase'''			,			lowerCAmelCase__	) != do_lower_case
										    or normalizer_state.get('''strip_accents'''			,			lowerCAmelCase__	) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''			,			lowerCAmelCase__	) != tokenize_chinese_chars
										):
																__SCREAMING_SNAKE_CASE						:     str					=		getattr(lowerCAmelCase__			,			normalizer_state.pop('''type'''	)	)
																__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		do_lower_case
																__SCREAMING_SNAKE_CASE						:     Dict					=		strip_accents
																__SCREAMING_SNAKE_CASE						:     List[str]					=		tokenize_chinese_chars
																__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		normalizer_class(**lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     List[Any]					=		do_lower_case


				def 		__magic_name__(       self			:Union[str, Any]			,			lowerCAmelCase__			:Optional[Any]			,			lowerCAmelCase__			:List[str]=None	)		->			List[str]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output


				def 		__magic_name__(       self			:Dict			,			lowerCAmelCase__			:List[int]			,			lowerCAmelCase__			:Optional[List[int]] = None	)		->			List[int]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		[self.sep_token_id]
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep	) * [0]
										return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



				def 		__magic_name__(       self			:List[str]			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Optional[str] = None	)		->			Tuple[str]:
										__SCREAMING_SNAKE_CASE						:     Dict					=		self._tokenizer.model.save(lowerCAmelCase__			,			name=lowerCAmelCase__	)
										return tuple(lowerCAmelCase__	)




from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:NestedDataStructureLike[PathLike]			,			lowerCAmelCase__			:Optional[NamedSplit] = None			,			lowerCAmelCase__			:Optional[Features] = None			,			lowerCAmelCase__			:str = None			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:bool = False			,			lowerCAmelCase__			:Optional[int] = None			,			**lowerCAmelCase__			:Optional[int]			,			)		->			Tuple:
										super().__init__(
										    lowerCAmelCase__			,			split=lowerCAmelCase__			,			features=lowerCAmelCase__			,			cache_dir=lowerCAmelCase__			,			keep_in_memory=lowerCAmelCase__			,			streaming=lowerCAmelCase__			,			num_proc=lowerCAmelCase__			,			**lowerCAmelCase__			,			)
										__SCREAMING_SNAKE_CASE						:     List[str]					=		path_or_paths if isinstance(lowerCAmelCase__			,			lowerCAmelCase__	) else {self.split: path_or_paths}
										__SCREAMING_SNAKE_CASE						:     int					=		Text(
										    cache_dir=lowerCAmelCase__			,			data_files=lowerCAmelCase__			,			features=lowerCAmelCase__			,			**lowerCAmelCase__			,			)



				def 		__magic_name__(       self			:Dict	)		->			Tuple:
										# Build iterable dataset
										if self.streaming:
																__SCREAMING_SNAKE_CASE						:     int					=		self.builder.as_streaming_dataset(split=self.split	)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		None
																__SCREAMING_SNAKE_CASE						:     str					=		None
																__SCREAMING_SNAKE_CASE						:     Dict					=		None
																__SCREAMING_SNAKE_CASE						:     Tuple					=		None

																self.builder.download_and_prepare(
																    download_config=lowerCAmelCase__			,			download_mode=lowerCAmelCase__			,			verification_mode=lowerCAmelCase__			,			base_path=lowerCAmelCase__			,			num_proc=self.num_proc			,			)
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.builder.as_dataset(
																    split=self.split			,			verification_mode=lowerCAmelCase__			,			in_memory=self.keep_in_memory	)
										return dataset
