from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


UpperCAmelCase      		=					logging.get_logger(__name__)

UpperCAmelCase      		=					{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase      		=					{
    "vocab_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json",
    },
    "merges_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "openai/clip-vit-base-patch32": (
            "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase      		=					{
    "openai/clip-vit-base-patch32": 77,
}




class  __lowerCAmelCase	(							a_    ):



 '''simple docstring'''
 _lowerCamelCase    :						Optional[int]					     =			VOCAB_FILES_NAMES
 _lowerCamelCase    :						List[Any]					     =			PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase    :						List[str]					     =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase    :						List[str]					     =			['input_ids', 'attention_mask']
 _lowerCamelCase    :						int					     =			CLIPTokenizer

 def __init__(      self							:		List[Any]							,      lowerCamelCase							:		Union[str, Any]=None							,      lowerCamelCase							:		Union[str, Any]=None							,      lowerCamelCase							:		int=None							,      lowerCamelCase							:		List[str]="<|endoftext|>"							,      lowerCamelCase							:		Optional[Any]="<|startoftext|>"							,      lowerCamelCase							:		Dict="<|endoftext|>"							,      lowerCamelCase							:		Any="<|endoftext|>"							,      **lowerCamelCase							:		str							,      )			->				Tuple:
        super().__init__(
            lowerCamelCase_							,      lowerCamelCase_							,      tokenizer_file=lowerCamelCase_							,      unk_token=lowerCamelCase_							,      bos_token=lowerCamelCase_							,      eos_token=lowerCamelCase_							,      pad_token=lowerCamelCase_							,      **lowerCamelCase_							,      )

        if not isinstance(self.backend_tokenizer.pre_tokenizer							,      pre_tokenizers.Sequence	):
               raise ValueError(
                   'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
                   ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
                   ' to be compatible with this version.The easiest way to do so is'
                   ' `CLIPTokenizerFast.from_pretrained(\"path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
                   ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
                   ' transformers.'	)

        self._wrap_decode_method_backend_tokenizer()

 def        __snake_case       (      self							:		List[str]	)			->				Union[str, Any]:
        _UpperCamelCase  :							Optional[Any]	    =	self.backend_tokenizer.decode

        def new_decode_method(*lowerCamelCase							:		Union[str, Any]							,      **lowerCamelCase							:		str	):
               _UpperCamelCase  :							Tuple	    =	orig_decode_method(*lowerCamelCase_							,      **lowerCamelCase_	)
               _UpperCamelCase  :							List[str]	    =	text.replace(self.backend_tokenizer.model.end_of_word_suffix							,      ' '	).strip()
               return text

        _UpperCamelCase  :							Any	    =	new_decode_method

 def        __snake_case       (      self							:		Optional[int]							,      lowerCamelCase							:		Optional[int]							,      lowerCamelCase							:		Tuple = None	)			->				List[Any]:
        _UpperCamelCase  :							Tuple	    =	[self.bos_token_id]
        _UpperCamelCase  :							Tuple	    =	[self.eos_token_id]

        if token_ids_a is None:
               return bos_token + token_ids_a + eos_token
        return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token

 def        __snake_case       (      self							:		List[Any]							,      lowerCamelCase							:		Union[str, Any]							,      lowerCamelCase							:		Dict = None	)			->				List[Any]:
        _UpperCamelCase  :							str	    =	[self.bos_token_id]
        _UpperCamelCase  :							Optional[int]	    =	[self.eos_token_id]

        if token_ids_a is None:
               return len(bos_token + token_ids_a + eos_token	) * [0]
        return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token	) * [0]

 def        __snake_case       (      self							:		Dict							,      lowerCamelCase							:		Optional[Any]							,      lowerCamelCase							:		List[Any] = None	)			->				Any:
        _UpperCamelCase  :							Optional[int]	    =	self._tokenizer.model.save(lowerCamelCase_							,      name=lowerCamelCase_	)
        return tuple(lowerCamelCase_	)








import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
				import sqlitea

				import sqlalchemy


_lowercase:					Optional[Any]      =						datasets.utils.logging.get_logger(__name__)



@dataclass
class       _lowercase				(  datasets.BuilderConfig      ):





	"""simple docstring"""
	__A		      =				None
	__A		      =				None
	__A		      =				None
	__A		      =				True
	__A		      =				None
	__A		      =				None
	__A		      =				None
	__A		      =				10000
	__A		      =				None




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						if self.sql is None:
											raise ValueError("sql must be specified"						)
						if self.con is None:
											raise ValueError("con must be specified"						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						,			):
						"""simple docstring"""

						a   	=  config_kwargs.copy()
						# We need to stringify the Selectable object to make its hash deterministic

						# The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
						a   	=  config_kwargs["sql"]
						if not isinstance(lowerCamelCase_						,			lowerCamelCase_						):
											if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
																import sqlalchemy

																if isinstance(lowerCamelCase_						,			sqlalchemy.sql.Selectable						):
																					a   	=  sqlalchemy.create_engine(config_kwargs["con"].split("://"						)[0] + "://"						)
																					a   	=  str(sql.compile(dialect=engine.dialect						)						)
																					a   	=  sql_str
																else:
																					raise TypeError(
																					    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(lowerCamelCase_						)}: {sql}'''						)
											else:
																raise TypeError(
																    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(lowerCamelCase_						)}: {sql}'''						)
						a   	=  config_kwargs["con"]
						if not isinstance(lowerCamelCase_						,			lowerCamelCase_						):
											a   	=  id(lowerCamelCase_						)
											logger.info(
											    F'''SQL connection \'con\' of type {type(lowerCamelCase_						)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''						)

						return super().create_config_id(lowerCamelCase_						,			custom_features=lowerCamelCase_						)




	@property
	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						a   	=  {
						    "index_col": self.index_col,
						    "columns": self.columns,
						    "params": self.params,
						    "coerce_float": self.coerce_float,
						    "parse_dates": self.parse_dates,
						}
						return pd_read_sql_kwargs




class       _lowercase				(  datasets.ArrowBasedBuilder      ):





	"""simple docstring"""
	__A		      =				SqlConfig




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return datasets.DatasetInfo(features=self.config.features						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,			gen_kwargs={}						)]




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if self.config.features is not None:
											a   	=  self.config.features.arrow_schema
											if all(not require_storage_cast(lowerCamelCase_						) for feature in self.config.features.values()						):
																# cheaper cast
																a   	=  pa.Table.from_arrays([pa_table[field.name] for field in schema]						,			schema=lowerCamelCase_						)
											else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																a   	=  table_cast(lowerCamelCase_						,			lowerCamelCase_						)
						return pa_table




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						a   	=  self.config.chunksize
						a   	=  pd.read_sql(
						    self.config.sql						,			self.config.con						,			chunksize=lowerCamelCase_						,			**self.config.pd_read_sql_kwargs						)
						a   	=  [sql_reader] if chunksize is None else sql_reader
						for chunk_idx, df in enumerate(lowerCamelCase_						):
											a   	=  pa.Table.from_pandas(lowerCamelCase_						)
											yield chunk_idx, self._cast_table(lowerCamelCase_						)
