

'''simple docstring'''

import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


UpperCAmelCase       =     'http://www.mocksite.com/file1.txt'
UpperCAmelCase       =     '"text": ["foo", "foo"]'
UpperCAmelCase       =     '6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'


class      lowerCAmelCase   :
       lowerCAmelCase_				=       2_0_0
       lowerCAmelCase_				=       {"""Content-Length""": """100"""}
       lowerCAmelCase_				=       {}







       def 	snake_case      (							self	:       Tuple  ,					**__lowercase	:       int					):







            """simple docstring"""







            return [bytes(__lowercase  ,					'utf-8'					)]






def  __UpperCamelCase							(		*lowercase__  :     Union[str, Any],    **lowercase__  :     Optional[Any]       ):






     '''simple docstring'''





     return MockResponse()

@pytest.mark.parametrize('urls_type',    [str, list, dict]       )
def  __UpperCamelCase							(		lowercase__  :     List[str],    lowercase__  :     int,    lowercase__  :     Dict       ):






     '''simple docstring'''





     import requests

     monkeypatch.setattr(lowercase__,    'request',    lowercase__       )

     __lowercase							=URL
     if issubclass(lowercase__,    lowercase__       ):
          __lowercase							=url
     elif issubclass(lowercase__,    lowercase__       ):
          __lowercase							=[url]
     elif issubclass(lowercase__,    lowercase__       ):
          __lowercase							={'train': url}
     __lowercase							='dummy'
     __lowercase							='downloads'
     __lowercase							=tmp_path
     __lowercase							=DownloadConfig(
         cache_dir=os.path.join(lowercase__,    lowercase__       ),    use_etag=lowercase__,    )
     __lowercase							=DownloadManager(dataset_name=lowercase__,    download_config=lowercase__       )
     __lowercase							=dl_manager.download(lowercase__       )
     __lowercase							=urls
     for downloaded_paths in [downloaded_paths]:
          if isinstance(lowercase__,    lowercase__       ):
               __lowercase							=[downloaded_paths]
               __lowercase							=[urls]
          elif isinstance(lowercase__,    lowercase__       ):
               assert "train" in downloaded_paths.keys()
               __lowercase							=downloaded_paths.values()
               __lowercase							=urls.values()
          assert downloaded_paths
          for downloaded_path, input_url in zip(lowercase__,    lowercase__       ):
               assert downloaded_path == dl_manager.downloaded_paths[input_url]
               __lowercase							=Path(lowercase__       )
               __lowercase							=downloaded_path.parts
               assert parts[-1] == HASH
               assert parts[-2] == cache_subdir
               assert downloaded_path.exists()
               __lowercase							=downloaded_path.read_text()
               assert content == CONTENT
               __lowercase							=downloaded_path.with_suffix('.json'       )
               assert metadata_downloaded_path.exists()
               __lowercase							=json.loads(metadata_downloaded_path.read_text()       )
               assert metadata_content == {"url": URL, "etag": None}

@pytest.mark.parametrize('paths_type',    [str, list, dict]       )
def  __UpperCamelCase							(		lowercase__  :     Dict,    lowercase__  :     Any,    lowercase__  :     Dict       ):






     '''simple docstring'''





     __lowercase							=str(lowercase__       )
     if issubclass(lowercase__,    lowercase__       ):
          __lowercase							=filename
     elif issubclass(lowercase__,    lowercase__       ):
          __lowercase							=[filename]
     elif issubclass(lowercase__,    lowercase__       ):
          __lowercase							={'train': filename}
     __lowercase							='dummy'
     __lowercase							=xz_file.parent
     __lowercase							='extracted'
     __lowercase							=DownloadConfig(
         cache_dir=lowercase__,    use_etag=lowercase__,    )
     __lowercase							=DownloadManager(dataset_name=lowercase__,    download_config=lowercase__       )
     __lowercase							=dl_manager.extract(lowercase__       )
     __lowercase							=paths
     for extracted_paths in [extracted_paths]:
          if isinstance(lowercase__,    lowercase__       ):
               __lowercase							=[extracted_paths]
               __lowercase							=[paths]
          elif isinstance(lowercase__,    lowercase__       ):
               assert "train" in extracted_paths.keys()
               __lowercase							=extracted_paths.values()
               __lowercase							=paths.values()
          assert extracted_paths
          for extracted_path, input_path in zip(lowercase__,    lowercase__       ):
               assert extracted_path == dl_manager.extracted_paths[input_path]
               __lowercase							=Path(lowercase__       )
               __lowercase							=extracted_path.parts
               assert parts[-1] == hash_url_to_filename(lowercase__,    etag=lowercase__       )
               assert parts[-2] == extracted_subdir
               assert extracted_path.exists()
               __lowercase							=extracted_path.read_text()
               __lowercase							=text_file.read_text()
               assert extracted_file_content == expected_file_content

def  __UpperCamelCase							(		lowercase__  :     Any,    lowercase__  :     List[Any]       ):






     '''simple docstring'''





     assert path.endswith('.jsonl'       )
     for num_items, line in enumerate(lowercase__,    start=1       ):
          __lowercase							=json.loads(line.decode('utf-8'       )       )
          assert item.keys() == {"col_1", "col_2", "col_3"}
     assert num_items == 4

@pytest.mark.parametrize('archive_jsonl',    ['tar_jsonl_path', 'zip_jsonl_path']       )
def  __UpperCamelCase							(		lowercase__  :     Optional[Any],    lowercase__  :     Optional[int]       ):






     '''simple docstring'''





     __lowercase							=request.getfixturevalue(lowercase__       )
     __lowercase							=DownloadManager()
     for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowercase__       ),    start=1       ):
          _test_jsonl(lowercase__,    lowercase__       )
     assert num_jsonl == 2

@pytest.mark.parametrize('archive_nested_jsonl',    ['tar_nested_jsonl_path', 'zip_nested_jsonl_path']       )
def  __UpperCamelCase							(		lowercase__  :     int,    lowercase__  :     List[str]       ):






     '''simple docstring'''





     __lowercase							=request.getfixturevalue(lowercase__       )
     __lowercase							=DownloadManager()
     for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowercase__       ),    start=1       ):
          for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowercase__       ),    start=1       ):
               _test_jsonl(lowercase__,    lowercase__       )
     assert num_tar == 1
     assert num_jsonl == 2







def  __UpperCamelCase							(		lowercase__  :     Any       ):






     '''simple docstring'''





     __lowercase							=DownloadManager()
     for num_file, file in enumerate(dl_manager.iter_files(lowercase__       ),    start=1       ):
          assert os.path.basename(lowercase__       ) == ("test.txt" if num_file == 1 else "train.txt")
     assert num_file == 2






'''simple docstring'''

import unittest

from transformers import DonutProcessor


lowerCamelCase							:   Tuple  						=	'naver-clova-ix/donut-base'
class   __lowerCAmelCase						(unittest.TestCase     ):





      '''simple docstring'''






      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= DonutProcessor.from_pretrained(UpperCamelCase )






      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= {
                 '''name''': '''John Doe''',
                 '''age''': '''99''',
                 '''city''': '''Atlanta''',
                 '''state''': '''GA''',
                 '''zip''': '''30301''',
                 '''phone''': '''123-4567''',
                 '''nicknames''': [{'''nickname''': '''Johnny'''}, {'''nickname''': '''JD'''}],
             }

             lowercase__      			= (
                 '''<s_name>John Doe</s_name><s_age>99</s_age><s_city>Atlanta</s_city>'''
                 '''<s_state>GA</s_state><s_zip>30301</s_zip><s_phone>123-4567</s_phone>'''
                 '''<s_nicknames><s_nickname>Johnny</s_nickname>'''
                 '''<sep/><s_nickname>JD</s_nickname></s_nicknames>'''
             )
             lowercase__      			= self.processor.tokenajson(UpperCamelCase )

             self.assertDictEqual(UpperCamelCase   ,						UpperCamelCase )




