






import copy
import random

from transformers import CLIPTokenizer





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	def __init__(							self       ,    *UpperCAmelCase       ,    **UpperCAmelCase    ):
							"""simple docstring"""







							super().__init__(*UpperCAmelCase       ,    **UpperCAmelCase    )
							_UpperCAmelCase									=			{}

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    *UpperCAmelCase       ,    **UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			super().add_tokens(UpperCAmelCase       ,    *UpperCAmelCase       ,    **UpperCAmelCase    )
							if num_added_tokens == 0:
													raise ValueError(
													    F"""The tokenizer already contains the token {placeholder_token}. Please pass a different"""
													    ' `placeholder_token` that is not already in the tokenizer.'    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    *UpperCAmelCase       ,    UpperCAmelCase=1       ,    **UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			[]
							if num_vec_per_token == 1:
													self.try_adding_tokens(UpperCAmelCase       ,    *UpperCAmelCase       ,    **UpperCAmelCase    )
													output.append(UpperCAmelCase    )
							else:
													_UpperCAmelCase									=			[]
													for i in range(UpperCAmelCase    ):
																			_UpperCAmelCase									=			placeholder_token + F"""_{i}"""
																			self.try_adding_tokens(UpperCAmelCase       ,    *UpperCAmelCase       ,    **UpperCAmelCase    )
																			output.append(UpperCAmelCase    )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
							for token in self.token_map:
													if token in placeholder_token:
																			raise ValueError(
																			    F"""The tokenizer already has placeholder token {token} that can get confused with"""
																			    F""" {placeholder_token}keep placeholder tokens independent"""    )
							_UpperCAmelCase									=			output

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase=False       ,    UpperCAmelCase=1.0    ):
							"""simple docstring"""







							if isinstance(UpperCAmelCase       ,    UpperCAmelCase    ):
													_UpperCAmelCase									=			[]
													for i in range(len(UpperCAmelCase    )    ):
																			output.append(self.replace_placeholder_tokens_in_text(text[i]       ,    vector_shuffle=UpperCAmelCase    )    )
													return output
							for placeholder_token in self.token_map:
													if placeholder_token in text:
																			_UpperCAmelCase									=			self.token_map[placeholder_token]
																			_UpperCAmelCase									=			tokens[: 1 + int(len(UpperCAmelCase    ) * prop_tokens_to_load    )]
																			if vector_shuffle:
																									_UpperCAmelCase									=			copy.copy(UpperCAmelCase    )
																									random.shuffle(UpperCAmelCase    )
																			_UpperCAmelCase									=			text.replace(UpperCAmelCase       ,    ' '.join(UpperCAmelCase    )    )
							return text

	def __call__(							self       ,    UpperCAmelCase       ,    *UpperCAmelCase       ,    UpperCAmelCase=False       ,    UpperCAmelCase=1.0       ,    **UpperCAmelCase    ):
							"""simple docstring"""







							return super().__call__(
							    self.replace_placeholder_tokens_in_text(
							        UpperCAmelCase       ,    vector_shuffle=UpperCAmelCase       ,    prop_tokens_to_load=UpperCAmelCase    )       ,    *UpperCAmelCase       ,    **UpperCAmelCase       ,    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    *UpperCAmelCase       ,    UpperCAmelCase=False       ,    UpperCAmelCase=1.0       ,    **UpperCAmelCase    ):
							"""simple docstring"""







							return super().encode(
							    self.replace_placeholder_tokens_in_text(
							        UpperCAmelCase       ,    vector_shuffle=UpperCAmelCase       ,    prop_tokens_to_load=UpperCAmelCase    )       ,    *UpperCAmelCase       ,    **UpperCAmelCase       ,    )








import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


_a      	=  get_logger(__name__)





class __lowerCamelCase (      enum.Enum):





	"""simple docstring"""






	UpperCamelCase__								=      "all_checks"
	UpperCamelCase__								=      "basic_checks"
	UpperCamelCase__								=      "no_checks"





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""









def __A    (      __lowerCAmelCase ,		__lowerCAmelCase ,		__lowerCAmelCase=None	)->  str:





						"""simple docstring"""



						if expected_checksums is None:
												logger.info('Unable to verify checksums.'	)
												return
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise ExpectedMoreDownloadedFiles(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise UnexpectedDownloadedFile(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						_UpperCAmelCase									=			[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
						_UpperCAmelCase									=			' for ' + verification_name if verification_name is not None else ''
						if len(__lowerCAmelCase	) > 0:
												raise NonMatchingChecksumError(
												    F"""Checksums didn't match{for_verification_name}:\n"""
												    F"""{bad_urls}\n"""
												    'Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'	)
						logger.info('All the checksums matched successfully' + for_verification_name	)





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""











class __lowerCamelCase (      snake_case__):





	"""simple docstring"""









def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  int:





						"""simple docstring"""



						if expected_splits is None:
												logger.info('Unable to verify splits sizes.'	)
												return
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise ExpectedMoreSplits(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						if len(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	) > 0:
												raise UnexpectedSplits(str(set(__lowerCAmelCase	) - set(__lowerCAmelCase	)	)	)
						_UpperCAmelCase									=			[
						    {'expected': expected_splits[name], 'recorded': recorded_splits[name]}
						    for name in expected_splits
						    if expected_splits[name].num_examples != recorded_splits[name].num_examples
						]
						if len(__lowerCAmelCase	) > 0:
												raise NonMatchingSplitsSizesError(str(__lowerCAmelCase	)	)
						logger.info('All the splits matched successfully.'	)


def __A    (      __lowerCAmelCase ,		__lowerCAmelCase = True	)->  dict:





						"""simple docstring"""



						if record_checksum:
												_UpperCAmelCase									=			shaaaa()
												with open(__lowerCAmelCase ,		'rb'	) as f:
																		for chunk in iter(lambda: f.read(1 << 20	) ,		b''	):
																								m.update(__lowerCAmelCase	)
																		_UpperCAmelCase									=			m.hexdigest()
						else:
												_UpperCAmelCase									=			None
						return {"num_bytes": os.path.getsize(__lowerCAmelCase	), "checksum": checksum}


def __A    (      __lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						if dataset_size and config.IN_MEMORY_MAX_SIZE:
												return dataset_size < config.IN_MEMORY_MAX_SIZE
						else:
												return False
