



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation

_lowercase:					Dict      =						logging.get_logger(__name__)

_lowercase:					str      =						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_lowercase:					Dict      =						{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

_lowercase:					List[str]      =						{
    "gpt2": 1024,
    "gpt2-medium": 1024,
    "gpt2-large": 1024,
    "gpt2-xl": 1024,
    "distilgpt2": 1024,
}






@lru_cache()
def        a( )						->			Optional[int]:
					"""simple docstring"""





					a   	=  (
					    list(range(ord("!"			)					,	ord("~"			) + 1			)			) + list(range(ord("¡"			)					,	ord("¬"			) + 1			)			) + list(range(ord("®"			)					,	ord("ÿ"			) + 1			)			)
					)
					a   	=  bs[:]
					a   	=  0
					for b in range(2**8			):
										if b not in bs:
															bs.append(A			)
															cs.append(2**8 + n			)
															n += 1
					a   	=  [chr(A			) for n in cs]
					return dict(zip(A					,	A			)			)






def        a( A			:       List[str]			)						->			Optional[Any]:
					"""simple docstring"""





					a   	=  set()
					a   	=  word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)			)
										a   	=  char
					return pairs







class       _lowercase				(  lowerCAmelCase      ):





	"""simple docstring"""
	__A		      =				VOCAB_FILES_NAMES
	__A		      =				PRETRAINED_VOCAB_FILES_MAP
	__A		      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__A		      =				["input_ids", "attention_mask"]




	def __init__(self						,			lowerCamelCase_						,			lowerCamelCase_						,			lowerCamelCase_="replace"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_="<|endoftext|>"						,			lowerCamelCase_=None						,			lowerCamelCase_=False						,			lowerCamelCase_=False						,			**lowerCamelCase_						,			):
						"""simple docstring"""

						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else bos_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else eos_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else unk_token
						a   	=  AddedToken(lowerCamelCase_						,			lstrip=lowerCamelCase_						,			rstrip=lowerCamelCase_						) if isinstance(lowerCamelCase_						,			lowerCamelCase_						) else pad_token
						super().__init__(
						    errors=lowerCamelCase_						,			unk_token=lowerCamelCase_						,			bos_token=lowerCamelCase_						,			eos_token=lowerCamelCase_						,			pad_token=lowerCamelCase_						,			add_prefix_space=lowerCamelCase_						,			add_bos_token=lowerCamelCase_						,			**lowerCamelCase_						,			)
						a   	=  add_bos_token

						with open(lowerCamelCase_						,			encoding="utf-8"						) as vocab_handle:
											a   	=  json.load(lowerCamelCase_						)
						a   	=  {v: k for k, v in self.encoder.items()}
						a   	=  errors  # how to handle errors in decoding
						a   	=  bytes_to_unicode()
						a   	=  {v: k for k, v in self.byte_encoder.items()}
						with open(lowerCamelCase_						,			encoding="utf-8"						) as merges_handle:
											a   	=  merges_handle.read().split("\n"						)[1:-1]
						a   	=  [tuple(merge.split()						) for merge in bpe_merges]
						a   	=  dict(zip(lowerCamelCase_						,			range(len(lowerCamelCase_						)						)						)						)
						a   	=  {}
						a   	=  add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						a   	=  re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"						)




	@property
	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return len(self.encoder						)




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return dict(self.encoder						,			**self.added_tokens_encoder						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if token in self.cache:
											return self.cache[token]
						a   	=  tuple(lowerCamelCase_						)
						a   	=  get_pairs(lowerCamelCase_						)

						if not pairs:
											return token

						while True:
											a   	=  min(lowerCamelCase_						,			key=lambda lowerCamelCase_						: self.bpe_ranks.get(lowerCamelCase_						,			float("inf"						)						)						)
											if bigram not in self.bpe_ranks:
																break
											a	,							a   	=  bigram
											a   	=  []
											a   	=  0
											while i < len(lowerCamelCase_						):
																try:
																					a   	=  word.index(lowerCamelCase_						,			lowerCamelCase_						)
																except ValueError:
																					new_word.extend(word[i:]						)
																					break
																else:
																					new_word.extend(word[i:j]						)
																					a   	=  j

																if word[i] == first and i < len(lowerCamelCase_						) - 1 and word[i + 1] == second:
																					new_word.append(first + second						)
																					i += 2
																else:
																					new_word.append(word[i]						)
																					i += 1
											a   	=  tuple(lowerCamelCase_						)
											a   	=  new_word
											if len(lowerCamelCase_						) == 1:
																break
											else:
																a   	=  get_pairs(lowerCamelCase_						)
						a   	=  " ".join(lowerCamelCase_						)
						a   	=  word
						return word




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_=None						):
						"""simple docstring"""

						if self.add_bos_token:
											a   	=  [self.bos_token_id]
						else:
											a   	=  []

						a   	=  bos_token_ids + token_ids_a

						if token_ids_a is None:
											return output

						return output + bos_token_ids + token_ids_a




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						,			lowerCamelCase_ = False						):
						"""simple docstring"""

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase_						,			token_ids_a=lowerCamelCase_						,			already_has_special_tokens=lowerCamelCase_						)

						if not self.add_bos_token:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase_						,			token_ids_a=lowerCamelCase_						,			already_has_special_tokens=lowerCamelCase_						)

						if token_ids_a is None:
											return [1] + ([0] * len(lowerCamelCase_						))
						return [1] + ([0] * len(lowerCamelCase_						)) + [1] + ([0] * len(lowerCamelCase_						))




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  []
						for token in re.findall(self.pat						,			lowerCamelCase_						):
											a   	=  "".join(
											    self.byte_encoder[b] for b in token.encode("utf-8"						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase_						).split(" "						)						)
						return bpe_tokens




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return self.encoder.get(lowerCamelCase_						,			self.encoder.get(self.unk_token						)						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return self.decoder.get(lowerCamelCase_						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  "".join(lowerCamelCase_						)
						a   	=  bytearray([self.byte_decoder[c] for c in text]						).decode("utf-8"						,			errors=self.errors						)
						return text




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						):
						"""simple docstring"""

						if not os.path.isdir(lowerCamelCase_						):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
						a   	=  os.path.join(
						    lowerCamelCase_						,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
						a   	=  os.path.join(
						    lowerCamelCase_						,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

						with open(lowerCamelCase_						,			"w"						,			encoding="utf-8"						) as f:
											f.write(json.dumps(self.encoder						,			indent=2						,			sort_keys=lowerCamelCase_						,			ensure_ascii=lowerCamelCase_						) + "\n"						)

						a   	=  0
						with open(lowerCamelCase_						,			"w"						,			encoding="utf-8"						) as writer:
											writer.write("#version: 0.2\n"						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,			key=lambda lowerCamelCase_						: kv[1]						):
																if index != token_index:
																					logger.warning(
																					    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																					    " Please check that the tokenizer is not corrupted!"						)
																					a   	=  token_index
																writer.write(" ".join(lowerCamelCase_						) + "\n"						)
																index += 1

						return vocab_file, merge_file




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_=False						,			**lowerCamelCase_						):
						"""simple docstring"""

						a   	=  kwargs.pop("add_prefix_space"						,			self.add_prefix_space						)
						if is_split_into_words or add_prefix_space:
											a   	=  " " + text
						return (text, kwargs)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						a   	=  []
						for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(lowerCamelCase_						,			add_special_tokens=lowerCamelCase_						) + [self.eos_token_id]						)
						if len(lowerCamelCase_						) > self.model_max_length:
											a   	=  input_ids[-self.model_max_length :]
						return input_ids





import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
				import sqlitea

				import sqlalchemy


_lowercase:					Optional[Any]      =						datasets.utils.logging.get_logger(__name__)



@dataclass
class       _lowercase				(  datasets.BuilderConfig      ):





	"""simple docstring"""
	__A		      =				None
	__A		      =				None
	__A		      =				None
	__A		      =				True
	__A		      =				None
	__A		      =				None
	__A		      =				None
	__A		      =				10000
	__A		      =				None




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						if self.sql is None:
											raise ValueError("sql must be specified"						)
						if self.con is None:
											raise ValueError("con must be specified"						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						,			lowerCamelCase_ = None						,			):
						"""simple docstring"""

						a   	=  config_kwargs.copy()
						# We need to stringify the Selectable object to make its hash deterministic

						# The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
						a   	=  config_kwargs["sql"]
						if not isinstance(lowerCamelCase_						,			lowerCamelCase_						):
											if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
																import sqlalchemy

																if isinstance(lowerCamelCase_						,			sqlalchemy.sql.Selectable						):
																					a   	=  sqlalchemy.create_engine(config_kwargs["con"].split("://"						)[0] + "://"						)
																					a   	=  str(sql.compile(dialect=engine.dialect						)						)
																					a   	=  sql_str
																else:
																					raise TypeError(
																					    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(lowerCamelCase_						)}: {sql}'''						)
											else:
																raise TypeError(
																    F'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(lowerCamelCase_						)}: {sql}'''						)
						a   	=  config_kwargs["con"]
						if not isinstance(lowerCamelCase_						,			lowerCamelCase_						):
											a   	=  id(lowerCamelCase_						)
											logger.info(
											    F'''SQL connection \'con\' of type {type(lowerCamelCase_						)} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''						)

						return super().create_config_id(lowerCamelCase_						,			custom_features=lowerCamelCase_						)




	@property
	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						a   	=  {
						    "index_col": self.index_col,
						    "columns": self.columns,
						    "params": self.params,
						    "coerce_float": self.coerce_float,
						    "parse_dates": self.parse_dates,
						}
						return pd_read_sql_kwargs




class       _lowercase				(  datasets.ArrowBasedBuilder      ):





	"""simple docstring"""
	__A		      =				SqlConfig




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						return datasets.DatasetInfo(features=self.config.features						)




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,			gen_kwargs={}						)]




	def 	UpperCamelCase_			(self						,			lowerCamelCase_						):
						"""simple docstring"""

						if self.config.features is not None:
											a   	=  self.config.features.arrow_schema
											if all(not require_storage_cast(lowerCamelCase_						) for feature in self.config.features.values()						):
																# cheaper cast
																a   	=  pa.Table.from_arrays([pa_table[field.name] for field in schema]						,			schema=lowerCamelCase_						)
											else:
																# more expensive cast; allows str <-> int/float or str to Audio for example
																a   	=  table_cast(lowerCamelCase_						,			lowerCamelCase_						)
						return pa_table




	def 	UpperCamelCase_			(self						):
						"""simple docstring"""

						a   	=  self.config.chunksize
						a   	=  pd.read_sql(
						    self.config.sql						,			self.config.con						,			chunksize=lowerCamelCase_						,			**self.config.pd_read_sql_kwargs						)
						a   	=  [sql_reader] if chunksize is None else sql_reader
						for chunk_idx, df in enumerate(lowerCamelCase_						):
											a   	=  pa.Table.from_pandas(lowerCamelCase_						)
											yield chunk_idx, self._cast_table(lowerCamelCase_						)
