



import copy
import random

from transformers import CLIPTokenizer

class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     int     ,		*__SCREAMING_SNAKE_CASE							:     List[Any]     ,		**__SCREAMING_SNAKE_CASE							:     List[Any]):


           '''simple docstring'''

           super().__init__(*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           __a						       =				{}



       def 		_lowerCamelCase			(		self							:     Tuple     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]     ,		*__SCREAMING_SNAKE_CASE							:     Any     ,		**__SCREAMING_SNAKE_CASE							:     Optional[Any]):


           '''simple docstring'''

           __a						       =				super().add_tokens(__SCREAMING_SNAKE_CASE     ,		*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
           if num_added_tokens == 0:
               raise ValueError(
                   F'The tokenizer already contains the token {placeholder_token}. Please pass a different'
                   ''' `placeholder_token` that is not already in the tokenizer.''')



       def 		_lowerCamelCase			(		self							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[str]     ,		*__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Any=1     ,		**__SCREAMING_SNAKE_CASE							:     Optional[Any]):


           '''simple docstring'''

           __a						       =				[]
           if num_vec_per_token == 1:
               self.try_adding_tokens(__SCREAMING_SNAKE_CASE     ,		*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
               output.append(__SCREAMING_SNAKE_CASE)
           else:
               __a						       =				[]
               for i in range(__SCREAMING_SNAKE_CASE):
                   __a						       =				placeholder_token + F'_{i}'
                   self.try_adding_tokens(__SCREAMING_SNAKE_CASE     ,		*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)
                   output.append(__SCREAMING_SNAKE_CASE)
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
           for token in self.token_map:
               if token in placeholder_token:
                   raise ValueError(
                       F'The tokenizer already has placeholder token {token} that can get confused with'
                       F' {placeholder_token}keep placeholder tokens independent')
           __a						       =				output



       def 		_lowerCamelCase			(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Tuple=False     ,		__SCREAMING_SNAKE_CASE							:     int=1.0):


           '''simple docstring'''

           if isinstance(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE):
               __a						       =				[]
               for i in range(len(__SCREAMING_SNAKE_CASE)):
                   output.append(self.replace_placeholder_tokens_in_text(text[i]     ,		vector_shuffle=__SCREAMING_SNAKE_CASE))
               return output
           for placeholder_token in self.token_map:
               if placeholder_token in text:
                   __a						       =				self.token_map[placeholder_token]
                   __a						       =				tokens[: 1 + int(len(__SCREAMING_SNAKE_CASE) * prop_tokens_to_load)]
                   if vector_shuffle:
                       __a						       =				copy.copy(__SCREAMING_SNAKE_CASE)
                       random.shuffle(__SCREAMING_SNAKE_CASE)
                   __a						       =				text.replace(__SCREAMING_SNAKE_CASE     ,		''' '''.join(__SCREAMING_SNAKE_CASE))
           return text



       def __call__(		self							:     str     ,		__SCREAMING_SNAKE_CASE							:     Optional[int]     ,		*__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Any=False     ,		__SCREAMING_SNAKE_CASE							:     int=1.0     ,		**__SCREAMING_SNAKE_CASE							:     int):


           '''simple docstring'''

           return super().__call__(
               self.replace_placeholder_tokens_in_text(
                   __SCREAMING_SNAKE_CASE     ,		vector_shuffle=__SCREAMING_SNAKE_CASE     ,		prop_tokens_to_load=__SCREAMING_SNAKE_CASE)     ,		*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)





       def 		_lowerCamelCase			(		self							:     List[Any]     ,		__SCREAMING_SNAKE_CASE							:     List[Any]     ,		*__SCREAMING_SNAKE_CASE							:     Any     ,		__SCREAMING_SNAKE_CASE							:     Any=False     ,		__SCREAMING_SNAKE_CASE							:     Union[str, Any]=1.0     ,		**__SCREAMING_SNAKE_CASE							:     Tuple):


           '''simple docstring'''

           return super().encode(
               self.replace_placeholder_tokens_in_text(
                   __SCREAMING_SNAKE_CASE     ,		vector_shuffle=__SCREAMING_SNAKE_CASE     ,		prop_tokens_to_load=__SCREAMING_SNAKE_CASE)     ,		*__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)










from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     Optional[int]     ,		__SCREAMING_SNAKE_CASE							:     NestedDataStructureLike[PathLike]     ,		__SCREAMING_SNAKE_CASE							:     Optional[NamedSplit] = None     ,		__SCREAMING_SNAKE_CASE							:     Optional[Features] = None     ,		__SCREAMING_SNAKE_CASE							:     str = None     ,		__SCREAMING_SNAKE_CASE							:     bool = False     ,		__SCREAMING_SNAKE_CASE							:     bool = False     ,		__SCREAMING_SNAKE_CASE							:     Optional[int] = None     ,		**__SCREAMING_SNAKE_CASE							:     List[str]     ,		):


           '''simple docstring'''

           super().__init__(
               __SCREAMING_SNAKE_CASE     ,		split=__SCREAMING_SNAKE_CASE     ,		features=__SCREAMING_SNAKE_CASE     ,		cache_dir=__SCREAMING_SNAKE_CASE     ,		keep_in_memory=__SCREAMING_SNAKE_CASE     ,		streaming=__SCREAMING_SNAKE_CASE     ,		num_proc=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)
           __a						       =				path_or_paths if isinstance(__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE) else {self.split: path_or_paths}
           __a						       =				Text(
               cache_dir=__SCREAMING_SNAKE_CASE     ,		data_files=__SCREAMING_SNAKE_CASE     ,		features=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE     ,		)





       def 		_lowerCamelCase			(		self							:     List[str]):


           '''simple docstring'''

           if self.streaming:
               __a						       =				self.builder.as_streaming_dataset(split=self.split)
           # Build regular (map-style) dataset
           else:
               __a						       =				None
               __a						       =				None
               __a						       =				None
               __a						       =				None

               self.builder.download_and_prepare(
                   download_config=__SCREAMING_SNAKE_CASE     ,		download_mode=__SCREAMING_SNAKE_CASE     ,		verification_mode=__SCREAMING_SNAKE_CASE     ,		base_path=__SCREAMING_SNAKE_CASE     ,		num_proc=self.num_proc     ,		)
               __a						       =				self.builder.as_dataset(
                   split=self.split     ,		verification_mode=__SCREAMING_SNAKE_CASE     ,		in_memory=self.keep_in_memory)
           return dataset





