






"""simple docstring"""



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_UpperCamelCase						:    str		=						datasets.utils.logging.get_logger(__name__)

@dataclass
class 			a	(  datasets.BuilderConfig      ):
    UpperCAmelCase_					:	Optional[datasets.Features]      	=None
    UpperCAmelCase_					:	str      	="utf-8"
    UpperCAmelCase_					:	Optional[str]      	=None
    UpperCAmelCase_					:	int      	=10 << 20  # 10MB
    UpperCAmelCase_					:	bool      	=False
    UpperCAmelCase_					:	str      	="line"


class 			a	(  datasets.ArrowBasedBuilder      ):
    UpperCAmelCase_					:	Tuple      	=TextConfig







    def      UpperCamelCase_ (     self			):
      return datasets.DatasetInfo(features=self.config.features			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      if not self.config.data_files:
        raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'			)
      lowercase				       =			dl_manager.download_and_extract(self.config.data_files			)
      if isinstance(__A				,						(str, list, tuple)			):
        lowercase				       =			data_files
        if isinstance(__A				,						__A			):
          lowercase				       =			[files]
        lowercase				       =			[dl_manager.iter_files(__A			) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,						gen_kwargs={'files': files}			)]
      lowercase				       =			[]
      for split_name, files in data_files.items():
        if isinstance(__A				,						__A			):
          lowercase				       =			[files]
        lowercase				       =			[dl_manager.iter_files(__A			) for file in files]
        splits.append(datasets.SplitGenerator(name=__A				,						gen_kwargs={'files': files}			)			)
      return splits







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      if self.config.features is not None:
        lowercase				       =			self.config.features.arrow_schema
        if all(not require_storage_cast(__A			) for feature in self.config.features.values()			):
          # cheaper cast
          lowercase				       =			pa_table.cast(__A			)
        else:
          # more expensive cast; allows str <-> int/float or str to Audio for example
          lowercase				       =			table_cast(__A				,						__A			)
        return pa_table
      else:
        return pa_table.cast(pa.schema({'text': pa.string()}			)			)







    def      UpperCamelCase_ (     self				,						_lowerCamelCase			):
      lowercase				       =			list(self.config.features			) if self.config.features is not None else ["""text"""]
      for file_idx, file in enumerate(itertools.chain.from_iterable(__A			)			):
        # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
        with open(__A				,						encoding=self.config.encoding				,						errors=self.config.errors			) as f:
          if self.config.sample_by == "line":
            lowercase				       =			0
            while True:
              lowercase				       =			f.read(self.config.chunksize			)
              if not batch:
                break
              batch += f.readline()  # finish current line
              # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
              lowercase				       =			StringIO(__A			).readlines()
              if not self.config.keep_linebreaks:
                lowercase				       =			[line.rstrip('\n'			) for line in batch]
              lowercase				       =			pa.Table.from_arrays([pa.array(__A			)]				,						names=__A			)
              # Uncomment for debugging (will print the Arrow table size and elements)
              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
              yield (file_idx, batch_idx), self._cast_table(__A			)
              batch_idx += 1
          elif self.config.sample_by == "paragraph":
            lowercase				       =			0
            lowercase				       =			""""""
            while True:
              lowercase				       =			f.read(self.config.chunksize			)
              if not new_batch:
                break
              batch += new_batch
              batch += f.readline()  # finish current line
              lowercase				       =			batch.split('\n\n'			)
              lowercase				       =			pa.Table.from_arrays(
                  [pa.array([example for example in batch[:-1] if example]			)]				,						names=__A			)
              # Uncomment for debugging (will print the Arrow table size and elements)
              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
              yield (file_idx, batch_idx), self._cast_table(__A			)
              batch_idx += 1
              lowercase				       =			batch[-1]
            if batch:
              lowercase				       =			pa.Table.from_arrays([pa.array([batch]			)]				,						names=__A			)
              yield (file_idx, batch_idx), self._cast_table(__A			)
          elif self.config.sample_by == "document":
            lowercase				       =			f.read()
            lowercase				       =			pa.Table.from_arrays([pa.array([text]			)]				,						names=__A			)
            yield file_idx, self._cast_table(__A			)




"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class      _SCREAMING_SNAKE_CASE   (     A__      ,					unittest.TestCase				):
			UpperCAmelCase_    :List[str]												=		GPTNeoXJapaneseTokenizer
			UpperCAmelCase_    :List[str]												=		False
			UpperCAmelCase_    :int												=		{"do_clean_text": False, "add_prefix_space": False}






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						super().setUp()

						lowerCAmelCase_       :int     	=    [
						    """こん""",
						    """こんに""",
						    """にちは""",
						    """ばんは""",
						    """世界,㔺界""",
						    """、""",
						    """。""",
						    """<BR>""",
						    """<SP>""",
						    """<TAB>""",
						    """<URL>""",
						    """<EMAIL>""",
						    """<TEL>""",
						    """<DATE>""",
						    """<PRICE>""",
						    """<BLOCK>""",
						    """<KIGOU>""",
						    """<U2000U2BFF>""",
						    """<|emoji1|>""",
						    """<unk>""",
						    """<|startoftext|>""",
						    """<|endoftext|>""",
						]
						lowerCAmelCase_       :str     	=    {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
						lowerCAmelCase_       :Dict     	=    {"""unk_token""": """<unk>"""}

						lowerCAmelCase_       :Union[str, Any]     	=    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["""vocab_file"""]			)
						lowerCAmelCase_       :Union[str, Any]     	=    os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["""emoji_file"""]			)
						with open(self.vocab_file			,  """w"""			,  encoding="""utf-8"""			) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)
						with open(self.emoji_file			,  """w"""			) as emoji_writer:
									emoji_writer.write(json.dumps(__A			)			)






			def 					__lowerCAmelCase					(   self			,  **__A			)     ->	str:
						kwargs.update(self.special_tokens_map			)
						return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[Any]:
						lowerCAmelCase_       :Optional[Any]     	=    """こんにちは、世界。 \nこんばんは、㔺界。😀"""
						lowerCAmelCase_       :List[Any]     	=    """こんにちは、世界。 \nこんばんは、世界。😀"""
						return input_text, output_text






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[Any]:
						lowerCAmelCase_	,			lowerCAmelCase_       :Optional[int]     	=    self.get_input_output_texts(__A			)
						lowerCAmelCase_       :List[Any]     	=    tokenizer.encode(__A			,  add_special_tokens=__A			)
						lowerCAmelCase_       :Optional[Any]     	=    tokenizer.decode(__A			,  clean_up_tokenization_spaces=__A			)
						return text, ids






			def 					__lowerCAmelCase					(   self			)     ->	str:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Any:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Optional[int]:
						pass  # TODO add if relevant






			def 					__lowerCAmelCase					(   self			)     ->	Optional[Any]:
						lowerCAmelCase_       :int     	=    self.get_tokenizer()

						# Testing tokenization
						lowerCAmelCase_       :List[str]     	=    """こんにちは、世界。　こんばんは、㔺界。"""
						lowerCAmelCase_       :int     	=    ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
						lowerCAmelCase_       :List[Any]     	=    tokenizer.tokenize(__A			)
						self.assertListEqual(__A			,  __A			)

						# Testing conversion to ids without special tokens
						lowerCAmelCase_       :List[str]     	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
						lowerCAmelCase_       :Dict     	=    tokenizer.convert_tokens_to_ids(__A			)
						self.assertListEqual(__A			,  __A			)

						# Testing conversion to ids with special tokens
						lowerCAmelCase_       :Tuple     	=    tokens + [tokenizer.unk_token]
						lowerCAmelCase_       :str     	=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
						lowerCAmelCase_       :Any     	=    tokenizer.convert_tokens_to_ids(__A			)
						self.assertListEqual(__A			,  __A			)






			@slow
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :str     	=    self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""			)

						lowerCAmelCase_       :Union[str, Any]     	=    tokenizer.encode("""ありがとう。"""			,  add_special_tokens=__A			)
						lowerCAmelCase_       :Dict     	=    tokenizer.encode("""どういたしまして。"""			,  add_special_tokens=__A			)

						lowerCAmelCase_       :Union[str, Any]     	=    tokenizer.build_inputs_with_special_tokens(__A			)
						lowerCAmelCase_       :Optional[int]     	=    tokenizer.build_inputs_with_special_tokens(__A			,  __A			)

						assert encoded_sentence == ids_a
						assert encoded_pair == ids_a + ids_a






			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						# Intentionally convert some words to accommodate character fluctuations unique to Japanese
						pass






			def 					__lowerCAmelCase					(   self			)     ->	List[str]:
						# tokenizer has no padding token
						pass






