



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_		:  Tuple										=  datasets.utils.logging.get_logger(__name__)



@dataclass
class 		_lowercase		(							datasets.BuilderConfig						):
   _UpperCAmelCase  						=      None
   _UpperCAmelCase  						=      "utf-8"
   _UpperCAmelCase  						=      None
   _UpperCAmelCase  						=      10 << 20  # 10MB
   _UpperCAmelCase  						=      False
   _UpperCAmelCase  						=      "line"



class 		_lowercase		(							datasets.ArrowBasedBuilder						):
   _UpperCAmelCase  						=      TextConfig

   def 		A				(   self      :							List[Any]				)							->					Union[str, Any]:


    """simple docstring"""






    return datasets.DatasetInfo(features=self.config.features				)

   def 		A				(   self      :							str				, __lowerCAmelCase      :							int				)							->					Union[str, Any]:


    """simple docstring"""






    if not self.config.data_files:
     raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""				)
    a							=		dl_manager.download_and_extract(self.config.data_files				)
    if isinstance(__lowerCAmelCase				, (str, list, tuple)				):
     a							=		data_files
     if isinstance(__lowerCAmelCase				, __lowerCAmelCase				):
      a							=		[files]
     a							=		[dl_manager.iter_files(__lowerCAmelCase				) for file in files]
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={"files": files}				)]
    a							=		[]
    for split_name, files in data_files.items():
     if isinstance(__lowerCAmelCase				, __lowerCAmelCase				):
      a							=		[files]
     a							=		[dl_manager.iter_files(__lowerCAmelCase				) for file in files]
     splits.append(datasets.SplitGenerator(name=__lowerCAmelCase				, gen_kwargs={"files": files}				)				)
    return splits

   def 		A				(   self      :							Optional[int]				, __lowerCAmelCase      :							pa.Table				)							->					pa.Table:


    """simple docstring"""






    if self.config.features is not None:
     a							=		self.config.features.arrow_schema
     if all(not require_storage_cast(__lowerCAmelCase				) for feature in self.config.features.values()				):
      # cheaper cast
      a							=		pa_table.cast(__lowerCAmelCase				)
     else:
      # more expensive cast; allows str <-> int/float or str to Audio for example
      a							=		table_cast(__lowerCAmelCase				, __lowerCAmelCase				)
     return pa_table
    else:
     return pa_table.cast(pa.schema({"text": pa.string()}				)				)




   def 		A				(   self      :							int				, __lowerCAmelCase      :							str				)							->					List[Any]:


    """simple docstring"""






    a							=		list(self.config.features				) if self.config.features is not None else ["text"]
    for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase				)				):
     # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
     with open(__lowerCAmelCase				, encoding=self.config.encoding				, errors=self.config.errors				) as f:
      if self.config.sample_by == "line":
       a							=		0
       while True:
        a							=		f.read(self.config.chunksize				)
        if not batch:
         break
        batch += f.readline()  # finish current line
        # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
        a							=		StringIO(__lowerCAmelCase				).readlines()
        if not self.config.keep_linebreaks:
         a							=		[line.rstrip("\n"				) for line in batch]
        a							=		pa.Table.from_arrays([pa.array(__lowerCAmelCase				)]				, names=__lowerCAmelCase				)
        # Uncomment for debugging (will print the Arrow table size and elements)
        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
        batch_idx += 1
      elif self.config.sample_by == "paragraph":
       a							=		0
       a							=		""
       while True:
        a							=		f.read(self.config.chunksize				)
        if not new_batch:
         break
        batch += new_batch
        batch += f.readline()  # finish current line
        a							=		batch.split("\n\n"				)
        a							=		pa.Table.from_arrays(
            [pa.array([example for example in batch[:-1] if example]				)]				, names=__lowerCAmelCase				)
        # Uncomment for debugging (will print the Arrow table size and elements)
        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
        batch_idx += 1
        a							=		batch[-1]
       if batch:
        a							=		pa.Table.from_arrays([pa.array([batch]				)]				, names=__lowerCAmelCase				)
        yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase				)
      elif self.config.sample_by == "document":
       a							=		f.read()
       a							=		pa.Table.from_arrays([pa.array([text]				)]				, names=__lowerCAmelCase				)
       yield file_idx, self._cast_table(__lowerCAmelCase				)









import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowerCAmelCase_     =			logging.get_logger(__name__)

lowerCAmelCase_     =			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase_     =			{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

lowerCAmelCase_     =			{
    'openai-gpt': 512,
}







def   snake_case		(				UpperCAmelCase						:							Union[str, Any] ):
				A									=   text.strip()
				if not text:
								return []
				A									=   text.split()
				return tokens
class   UpperCamelCase			(					snake_case__			):







		"""simple docstring"""



		def __init__(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]=True    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]=None    ,_SCREAMING_SNAKE_CASE		:							int=True    ,_SCREAMING_SNAKE_CASE		:							Any=None    ,_SCREAMING_SNAKE_CASE		:							str=True    ,)     -> List[Any]:


						'''simple docstring'''
						if never_split is None:
										A									=   []
						A									=   do_lower_case
						A									=   set(_SCREAMING_SNAKE_CASE		)
						A									=   tokenize_chinese_chars
						A									=   strip_accents
						A									=   do_split_on_punc
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							int    ,_SCREAMING_SNAKE_CASE		:							Optional[int]=None		)     -> int:


						'''simple docstring'''
						A									=   self.never_split.union(set(_SCREAMING_SNAKE_CASE		)		) if never_split else self.never_split
						A									=   self._clean_text(_SCREAMING_SNAKE_CASE		)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
										A									=   self._tokenize_chinese_chars(_SCREAMING_SNAKE_CASE		)
						# prevents treating the same character with different unicode codepoints as different characters
						A									=   unicodedata.normalize('NFC'    ,_SCREAMING_SNAKE_CASE		)
						A									=   whitespace_tokenize(_SCREAMING_SNAKE_CASE		)
						A									=   []
						for token in orig_tokens:
										if token not in never_split:
														if self.do_lower_case:
																		A									=   token.lower()
																		if self.strip_accents is not False:
																						A									=   self._run_strip_accents(_SCREAMING_SNAKE_CASE		)
														elif self.strip_accents:
																		A									=   self._run_strip_accents(_SCREAMING_SNAKE_CASE		)
										split_tokens.extend(self._run_split_on_punc(_SCREAMING_SNAKE_CASE    ,_SCREAMING_SNAKE_CASE		)		)

						A									=   whitespace_tokenize(' '.join(_SCREAMING_SNAKE_CASE		)		)
						return output_tokens
		def    A(	self		:							int    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Optional[Any]:


						'''simple docstring'''
						A									=   unicodedata.normalize('NFD'    ,_SCREAMING_SNAKE_CASE		)
						A									=   []
						for char in text:
										A									=   unicodedata.category(_SCREAMING_SNAKE_CASE		)
										if cat == "Mn":
														continue
										output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)
		def    A(	self		:							Tuple    ,_SCREAMING_SNAKE_CASE		:							List[str]    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]=None		)     -> int:


						'''simple docstring'''
						if not self.do_split_on_punc or (never_split is not None and text in never_split):
										return [text]
						A									=   list(_SCREAMING_SNAKE_CASE		)
						A									=   0
						A									=   True
						A									=   []
						while i < len(_SCREAMING_SNAKE_CASE		):
										A									=   chars[i]
										if _is_punctuation(_SCREAMING_SNAKE_CASE		):
														output.append([char]		)
														A									=   True
										else:
														if start_new_word:
																		output.append([]		)
														A									=   False
														output[-1].append(_SCREAMING_SNAKE_CASE		)
										i += 1

						return ["".join(_SCREAMING_SNAKE_CASE		) for x in output]
		def    A(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> Optional[int]:


						'''simple docstring'''
						A									=   []
						for char in text:
										A									=   ord(_SCREAMING_SNAKE_CASE		)
										if self._is_chinese_char(_SCREAMING_SNAKE_CASE		):
														output.append(' '		)
														output.append(_SCREAMING_SNAKE_CASE		)
														output.append(' '		)
										else:
														output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)
		def    A(	self		:							Dict    ,_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Tuple:


						'''simple docstring'''
						# This defines a "chinese character" as anything in the CJK Unicode block:
						#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
						#
						# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
						# despite its name. The modern Korean Hangul alphabet is a different block,
						# as is Japanese Hiragana and Katakana. Those alphabets are used to write
						# space-separated words, so they are not treated specially and handled
						# like the all of the other languages.
						if (
						    (cp >= 0x4e00 and cp <= 0x9fff)
						    or (cp >= 0x3400 and cp <= 0x4dbf)  #
						    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
						    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
						    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
						    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
						    or (cp >= 0xf900 and cp <= 0xfaff)
						    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
						):  #
										return True

						return False







		def    A(	self		:							Union[str, Any]    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> List[str]:


						'''simple docstring'''
						A									=   []
						for char in text:
										A									=   ord(_SCREAMING_SNAKE_CASE		)
										if cp == 0 or cp == 0xfffd or _is_control(_SCREAMING_SNAKE_CASE		):
														continue
										if _is_whitespace(_SCREAMING_SNAKE_CASE		):
														output.append(' '		)
										else:
														output.append(_SCREAMING_SNAKE_CASE		)
						return "".join(_SCREAMING_SNAKE_CASE		)







def   snake_case		(				UpperCAmelCase						:							str ):
				A									=   set()
				A									=   word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								A									=   char
				return pairs







def   snake_case		(				UpperCAmelCase						:							str ):
				A									=   text.replace('—',    '-' )
				A									=   text.replace('–',    '-' )
				A									=   text.replace('―',    '-' )
				A									=   text.replace('…',    '...' )
				A									=   text.replace('´',    '\'' )
				A									=   re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)',    r' \1 ',    UpperCAmelCase )
				A									=   re.sub(r'\s*\n\s*',    ' \n ',    UpperCAmelCase )
				A									=   re.sub(r'[^\S\n]+',    ' ',    UpperCAmelCase )
				return text.strip()



class   UpperCamelCase			(					snake_case__			):







		"""simple docstring"""



		snake_case			 = VOCAB_FILES_NAMES
		snake_case			 = PRETRAINED_VOCAB_FILES_MAP
		snake_case			 = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case			 = ["input_ids", "attention_mask"]
		def __init__(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							Union[str, Any]    ,_SCREAMING_SNAKE_CASE		:							Dict    ,_SCREAMING_SNAKE_CASE		:							Dict="<unk>"    ,**_SCREAMING_SNAKE_CASE		:							Optional[Any]		)     -> Dict:


						'''simple docstring'''
						super().__init__(unk_token=_SCREAMING_SNAKE_CASE    ,**_SCREAMING_SNAKE_CASE		)

						try:
										import ftfy
										from spacy.lang.en import English

										A									=   English()
										A									=   _nlp.tokenizer
										A									=   ftfy.fix_text
						except ImportError:
										logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'		)
										A									=   BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		)
										A									=   None

						with open(_SCREAMING_SNAKE_CASE    ,encoding='utf-8'		) as vocab_handle:
										A									=   json.load(_SCREAMING_SNAKE_CASE		)
						A									=   {v: k for k, v in self.encoder.items()}
						with open(_SCREAMING_SNAKE_CASE    ,encoding='utf-8'		) as merges_handle:
										A									=   merges_handle.read().split('\n'		)[1:-1]
						A									=   [tuple(merge.split()		) for merge in merges]
						A									=   dict(zip(_SCREAMING_SNAKE_CASE    ,range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
						A									=   {}
		@property
		def    A(	self		:							Union[str, Any]		)     -> Dict:


						'''simple docstring'''
						return True
		@property
		def    A(	self		:							Optional[int]		)     -> str:


						'''simple docstring'''
						return len(self.encoder		)
		def    A(	self		:							Optional[Any]		)     -> str:


						'''simple docstring'''
						return dict(self.encoder    ,**self.added_tokens_encoder		)
		def    A(	self		:							Optional[int]    ,_SCREAMING_SNAKE_CASE		:							Dict		)     -> List[Any]:


						'''simple docstring'''
						A									=   tuple(token[:-1]		) + (token[-1] + '</w>',)
						if token in self.cache:
										return self.cache[token]
						A									=   get_pairs(_SCREAMING_SNAKE_CASE		)

						if not pairs:
										return token + "</w>"

						while True:
										A									=   min(_SCREAMING_SNAKE_CASE    ,key=lambda _SCREAMING_SNAKE_CASE		: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE    ,float('inf'		)		)		)
										if bigram not in self.bpe_ranks:
														break
										A			,		A									=   bigram
										A									=   []
										A									=   0
										while i < len(_SCREAMING_SNAKE_CASE		):
														try:
																		A									=   word.index(_SCREAMING_SNAKE_CASE    ,_SCREAMING_SNAKE_CASE		)
														except ValueError:
																		new_word.extend(word[i:]		)
																		break
														else:
																		new_word.extend(word[i:j]		)
																		A									=   j

														if word[i] == first and i < len(_SCREAMING_SNAKE_CASE		) - 1 and word[i + 1] == second:
																		new_word.append(first + second		)
																		i += 2
														else:
																		new_word.append(word[i]		)
																		i += 1
										A									=   tuple(_SCREAMING_SNAKE_CASE		)
										A									=   new_word
										if len(_SCREAMING_SNAKE_CASE		) == 1:
														break
										else:
														A									=   get_pairs(_SCREAMING_SNAKE_CASE		)
						A									=   ' '.join(_SCREAMING_SNAKE_CASE		)
						if word == "\n  </w>":
										A									=   '\n</w>'
						A									=   word
						return word
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							Optional[int]		)     -> List[Any]:


						'''simple docstring'''
						A									=   []
						if self.fix_text is None:
										# Using BERT's BasicTokenizer
										A									=   self.nlp.tokenize(_SCREAMING_SNAKE_CASE		)
										for token in text:
														split_tokens.extend(list(self.bpe(_SCREAMING_SNAKE_CASE		).split(' '		)		)		)
						else:
										# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
										A									=   self.nlp(text_standardize(self.fix_text(_SCREAMING_SNAKE_CASE		)		)		)
										for token in text:
														split_tokens.extend(list(self.bpe(token.text.lower()		).split(' '		)		)		)
						return split_tokens
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							List[Any]		)     -> Any:


						'''simple docstring'''
						return self.encoder.get(_SCREAMING_SNAKE_CASE    ,self.encoder.get(self.unk_token		)		)
		def    A(	self		:							str    ,_SCREAMING_SNAKE_CASE		:							List[str]		)     -> List[str]:


						'''simple docstring'''
						return self.decoder.get(_SCREAMING_SNAKE_CASE    ,self.unk_token		)
		def    A(	self		:							List[Any]    ,_SCREAMING_SNAKE_CASE		:							str		)     -> Any:


						'''simple docstring'''
						A									=   ''.join(_SCREAMING_SNAKE_CASE		).replace('</w>'    ,' '		).strip()
						return out_string







		def    A(	self		:							Tuple    ,_SCREAMING_SNAKE_CASE		:							str    ,_SCREAMING_SNAKE_CASE		:							Optional[str] = None		)     -> Tuple[str]:


						'''simple docstring'''
						if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
										return
						A									=   os.path.join(
						    _SCREAMING_SNAKE_CASE    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
						A									=   os.path.join(
						    _SCREAMING_SNAKE_CASE    ,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']		)

						with open(_SCREAMING_SNAKE_CASE    ,'w'    ,encoding='utf-8'		) as f:
										f.write(json.dumps(self.encoder    ,indent=2    ,sort_keys=_SCREAMING_SNAKE_CASE    ,ensure_ascii=_SCREAMING_SNAKE_CASE		) + '\n'		)

						A									=   0
						with open(_SCREAMING_SNAKE_CASE    ,'w'    ,encoding='utf-8'		) as writer:
										writer.write('#version: 0.2\n'		)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,key=lambda _SCREAMING_SNAKE_CASE		: kv[1]		):
														if index != token_index:
																		logger.warning(
																		    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																		    ' Please check that the tokenizer is not corrupted!'		)
																		A									=   token_index
														writer.write(' '.join(_SCREAMING_SNAKE_CASE		) + '\n'		)
														index += 1

						return vocab_file, merge_file





