
"""simple docstring"""






import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
UpperCamelCase_						   =   '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'

# TODO: Add description of the dataset here
# You can copy an official description
UpperCamelCase_						   =   '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'

# TODO: Add a link to an official homepage for the dataset here
UpperCamelCase_						   =   ''

# TODO: Add the licence for the dataset here if you can find it
UpperCamelCase_						   =   ''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
UpperCamelCase_						   =   {
    'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
    'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
}





class 							snake_case       (     datasets.GeneratorBasedBuilder		):
   a_					:		Tuple     					=     datasets.Version("""1.1.0"""		)

   # This is an example of a dataset with multiple configurations.
   # If you don't want/need to define several sub-sets in your dataset,
   # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

   # If you need to make complex sub-parts in the datasets with configurable options
   # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
   # BUILDER_CONFIG_CLASS = MyBuilderConfig

   # You will be able to load one or the other configurations in the following list with
   # data = datasets.load_dataset('my_dataset', 'first_domain')
   # data = datasets.load_dataset('my_dataset', 'second_domain')
   a_					:		Union[str, Any]     					=     [
       datasets.BuilderConfig(name="""first_domain"""							,       version=SCREAMING_SNAKE_CASE_							,       description="""This part of my dataset covers a first domain"""		),
       datasets.BuilderConfig(name="""second_domain"""							,       version=SCREAMING_SNAKE_CASE_							,       description="""This part of my dataset covers a second domain"""		),
   ]

   a_					:		List[str]     					=     """first_domain"""  # It's not mandatory to have a default configuration. Just use one if it make sense.







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
      if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
         a_						   =     datasets.Features(
             {
                 "sentence": datasets.Value("string"),
                 "option1": datasets.Value("string"),
                 "answer": datasets.Value("string")
                 # These are the features of your dataset like images, labels ...
             })
      else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
         a_						   =     datasets.Features(
             {
                 "sentence": datasets.Value("string"),
                 "option2": datasets.Value("string"),
                 "second_domain_answer": datasets.Value("string")
                 # These are the features of your dataset like images, labels ...
             })
      return datasets.DatasetInfo(
          # This is the description that will appear on the datasets page.
          description=_DESCRIPTION       ,							features=__UpperCAmelCase       ,							homepage=_HOMEPAGE       ,							license=_LICENSE       ,							citation=_CITATION       ,							)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Any:
      # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
      # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name

      # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
      # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
      # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
      a_						   =     _URLS[self.config.name]
      a_						   =     dl_manager.download_and_extract(__UpperCAmelCase)
      return [
          datasets.SplitGenerator(
              name=datasets.Split.TRAIN       ,							gen_kwargs={
                  "filepath": os.path.join(__UpperCAmelCase       ,							"train.jsonl"),
                  "split": "train",
              }       ,							),
          datasets.SplitGenerator(
              name=datasets.Split.VALIDATION       ,							gen_kwargs={
                  "filepath": os.path.join(__UpperCAmelCase       ,							"dev.jsonl"),
                  "split": "dev",
              }       ,							),
          datasets.SplitGenerator(
              name=datasets.Split.TEST       ,							gen_kwargs={
                  "filepath": os.path.join(__UpperCAmelCase       ,							"test.jsonl"),
                  "split": "test"
              }       ,							),
      ]







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							__UpperCAmelCase)   ->Optional[Any]:
      # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
      # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
      with open(__UpperCAmelCase       ,							encoding="utf-8") as f:
         for key, row in enumerate(__UpperCAmelCase):
            a_						   =     json.loads(__UpperCAmelCase)
            if self.config.name == "first_domain":
               # Yields examples as (key, example) tuples
               yield key, {
                   "sentence": data["sentence"],
                   "option1": data["option1"],
                   "answer": "" if split == "test" else data["answer"],
               }
            else:
               yield key, {
                   "sentence": data["sentence"],
                   "option2": data["option2"],
                   "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
               }

"""simple docstring"""






import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 							snake_case       (     SCREAMING_SNAKE_CASE_							,       unittest.TestCase		):
   a_					:		Optional[int]     					=     GPTNeoXJapaneseTokenizer
   a_					:		Dict     					=     False
   a_					:		Union[str, Any]     					=     {"""do_clean_text""": False, """add_prefix_space""": False}







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      super().setUp()

      a_						   =     [
          "こん",
          "こんに",
          "にちは",
          "ばんは",
          "世界,㔺界",
          "、",
          "。",
          "<BR>",
          "<SP>",
          "<TAB>",
          "<URL>",
          "<EMAIL>",
          "<TEL>",
          "<DATE>",
          "<PRICE>",
          "<BLOCK>",
          "<KIGOU>",
          "<U2000U2BFF>",
          "<|emoji1|>",
          "<unk>",
          "<|startoftext|>",
          "<|endoftext|>",
      ]
      a_						   =     {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
      a_						   =     {"unk_token": "<unk>"}

      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["vocab_file"])
      a_						   =     os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["emoji_file"])
      with open(self.vocab_file       ,							"w"       ,							encoding="utf-8") as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
      with open(self.emoji_file       ,							"w") as emoji_writer:
         emoji_writer.write(json.dumps(__UpperCAmelCase))







   def    UpperCAmelCase__	( self       ,							**__UpperCAmelCase)   ->int:
      kwargs.update(self.special_tokens_map)
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							**__UpperCAmelCase)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Dict:
      a_						   =     "こんにちは、世界。 \nこんばんは、㔺界。😀"
      a_						   =     "こんにちは、世界。 \nこんばんは、世界。😀"
      return input_text, output_text







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[Any]:
      a_   ,			a_						   =     self.get_input_output_texts(__UpperCAmelCase)
      a_						   =     tokenizer.encode(__UpperCAmelCase       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.decode(__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase)
      return text, ids







   def    UpperCAmelCase__	( self)   ->Any:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->int:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      pass  # TODO add if relevant







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     self.get_tokenizer()

      # Testing tokenization
      a_						   =     "こんにちは、世界。　こんばんは、㔺界。"
      a_						   =     ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
      a_						   =     tokenizer.tokenize(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids without special tokens
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)

      # Testing conversion to ids with special tokens
      a_						   =     tokens + [tokenizer.unk_token]
      a_						   =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      a_						   =     tokenizer.convert_tokens_to_ids(__UpperCAmelCase)
      self.assertListEqual(__UpperCAmelCase       ,							__UpperCAmelCase)







   @slow
   def    UpperCAmelCase__	( self)   ->Tuple:
      a_						   =     self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b")

      a_						   =     tokenizer.encode("ありがとう。"       ,							add_special_tokens=__UpperCAmelCase)
      a_						   =     tokenizer.encode("どういたしまして。"       ,							add_special_tokens=__UpperCAmelCase)

      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase)
      a_						   =     tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase       ,							__UpperCAmelCase)

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a







   def    UpperCAmelCase__	( self)   ->Tuple:
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      # tokenizer has no padding token
      pass