


'''simple docstring'''




import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
__snake_case    						=sys.version_info >= (3, 10)





def 	a_					( lowerCamelCase	:     List[Any]=None	, lowerCamelCase	:     Tuple=None				):
   return field(default_factory=lambda: default	, metadata=lowerCamelCase				)

@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	int
     lowerCamelCase				:	float
     lowerCamelCase				:	str
     lowerCamelCase				:	bool




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	int		=     42
     lowerCamelCase				:	str		=     field(default='''toto'''       ,	metadata={'''help''': '''help message'''}		)




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	bool		=     False
     lowerCamelCase				:	bool		=     True
     lowerCamelCase				:	Optional[bool]		=     None




class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	List[Any]		=     '''titi'''
     lowerCamelCase				:	List[str]		=     '''toto'''




class 		UpperCAmelCase_      (  __lowercase		):
     lowerCamelCase				:	List[str]		=     '''titi'''
     lowerCamelCase				:	Any		=     '''toto'''
     lowerCamelCase				:	Union[str, Any]		=     42




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	BasicEnum		=     "toto"





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Union[str, Any]:
        lowerCAmelCase     =			BasicEnum(self.foo    )




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	MixedTypeEnum		=     "toto"





     def 							__UpperCAmelCase      ( self       :      int    )				->  Dict:
        lowerCAmelCase     =			MixedTypeEnum(self.foo    )




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	Optional[int]		=     None
     lowerCamelCase				:	Optional[float]		=     field(default=__lowercase       ,	metadata={'''help''': '''help message'''}		)
     lowerCamelCase				:	Optional[str]		=     None
     lowerCamelCase				:	Optional[List[str]]		=     list_field(default=[]		)
     lowerCamelCase				:	Optional[List[int]]		=     list_field(default=[]		)




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	List[int]		=     list_field(default=[]		)
     lowerCamelCase				:	List[int]		=     list_field(default=[1, 2, 3]		)
     lowerCamelCase				:	List[str]		=     list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']		)
     lowerCamelCase				:	List[float]		=     list_field(default=[0.1, 0.2, 0.3]		)




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	List[int]		=     field()
     lowerCamelCase				:	str		=     field()
     lowerCamelCase				:	BasicEnum		=     field()





     def 							__UpperCAmelCase      ( self       :      List[str]    )				->  Optional[int]:
        lowerCAmelCase     =			BasicEnum(self.required_enum    )




@dataclass
class 		UpperCAmelCase_      :
     lowerCamelCase				:	int
     lowerCamelCase				:	"BasicEnum"		=     field()
     lowerCamelCase				:	"Optional[bool]"		=     None
     lowerCamelCase				:	"str"		=     field(default='''toto'''       ,	metadata={'''help''': '''help message'''}		)
     lowerCamelCase				:	"List[str]"		=     list_field(default=['''Hallo''', '''Bonjour''', '''Hello''']		)


if is_python_no_less_than_3_10:

       @dataclass
       class 		UpperCAmelCase_      :
            lowerCamelCase				:	bool		=     False
            lowerCamelCase				:	bool		=     True
            lowerCamelCase				:	bool | None		=     None




       @dataclass
       class 		UpperCAmelCase_      :
            lowerCamelCase				:	int | None		=     None
            lowerCamelCase				:	float | None		=     field(default=__lowercase       ,	metadata={'''help''': '''help message'''}		)
            lowerCamelCase				:	str | None		=     None
            lowerCamelCase				:	list[str] | None		=     list_field(default=[]		)
            lowerCamelCase				:	list[int] | None		=     list_field(default=[]		)




class 		UpperCAmelCase_      (  unittest.TestCase		):





     def 							__UpperCAmelCase      ( self       :      Any    ,      UpperCAmelCase__       :      argparse.ArgumentParser    ,      UpperCAmelCase__       :      argparse.ArgumentParser    )				->  Dict:
        self.assertEqual(len(a._actions    )    ,      len(b._actions    )    )
        for x, y in zip(a._actions    ,      b._actions    ):
           lowerCAmelCase     =			{k: v for k, v in vars(UpperCAmelCase__    ).items() if k != 'container'}
           lowerCAmelCase     =			{k: v for k, v in vars(UpperCAmelCase__    ).items() if k != 'container'}

           # Choices with mixed type have custom function as "type"
           # So we need to compare results directly for equality
           if xx.get('choices'    ,      UpperCAmelCase__    ) and yy.get('choices'    ,      UpperCAmelCase__    ):
              for expected_choice in yy["choices"] + xx["choices"]:
                 self.assertEqual(xx['type'](UpperCAmelCase__    )    ,      yy['type'](UpperCAmelCase__    )    )
              del xx["type"], yy["type"]

           self.assertEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    )				->  Tuple:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument('--bar'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument('--baz'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument('--flag'    ,      type=UpperCAmelCase__    ,      default=UpperCAmelCase__    ,      const=UpperCAmelCase__    ,      nargs='?'    )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

        lowerCAmelCase     =			['--foo', '1', '--baz', 'quux', '--bar', '0.5']
        ((lowerCAmelCase)     ,  )     =			parser.parse_args_into_dataclasses(UpperCAmelCase__    ,      look_for_args_file=UpperCAmelCase__    )
        self.assertFalse(example.flag    )





     def 							__UpperCAmelCase      ( self       :      int    )				->  int:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo'    ,      default=4_2    ,      type=UpperCAmelCase__    )
        expected.add_argument('--baz'    ,      default='toto'    ,      type=UpperCAmelCase__    ,      help='help message'    )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  str:
        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo'    ,      type=UpperCAmelCase__    ,      default=UpperCAmelCase__    ,      const=UpperCAmelCase__    ,      nargs='?'    )
        expected.add_argument('--baz'    ,      type=UpperCAmelCase__    ,      default=UpperCAmelCase__    ,      const=UpperCAmelCase__    ,      nargs='?'    )
        # A boolean no_* argument always has to come after its "default: True" regular counter-part
        # and its default must be set to False
        expected.add_argument('--no_baz'    ,      action='store_false'    ,      default=UpperCAmelCase__    ,      dest='baz'    )
        expected.add_argument('--opt'    ,      type=UpperCAmelCase__    ,      default=UpperCAmelCase__    )

        lowerCAmelCase     =			[WithDefaultBoolExample]
        if is_python_no_less_than_3_10:
           dataclass_types.append(UpperCAmelCase__    )

        for dataclass_type in dataclass_types:
           lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )
           self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

           lowerCAmelCase     =			parser.parse_args([]    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      opt=UpperCAmelCase__    )    )

           lowerCAmelCase     =			parser.parse_args(['--foo', '--no_baz']    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      opt=UpperCAmelCase__    )    )

           lowerCAmelCase     =			parser.parse_args(['--foo', '--baz']    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      opt=UpperCAmelCase__    )    )

           lowerCAmelCase     =			parser.parse_args(['--foo', 'True', '--baz', 'True', '--opt', 'True']    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      opt=UpperCAmelCase__    )    )

           lowerCAmelCase     =			parser.parse_args(['--foo', 'False', '--baz', 'False', '--opt', 'False']    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      opt=UpperCAmelCase__    )    )





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Union[str, Any]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument(
            '--foo'    ,      default='toto'    ,      choices=['titi', 'toto', 4_2]    ,      type=make_choice_type_function(['titi', 'toto', 4_2]    )    ,      )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

        lowerCAmelCase     =			parser.parse_args([]    )
        self.assertEqual(args.foo    ,      'toto'    )
        lowerCAmelCase     =			parser.parse_args_into_dataclasses([]    )[0]
        self.assertEqual(enum_ex.foo    ,      MixedTypeEnum.toto    )

        lowerCAmelCase     =			parser.parse_args(['--foo', 'titi']    )
        self.assertEqual(args.foo    ,      'titi'    )
        lowerCAmelCase     =			parser.parse_args_into_dataclasses(['--foo', 'titi']    )[0]
        self.assertEqual(enum_ex.foo    ,      MixedTypeEnum.titi    )

        lowerCAmelCase     =			parser.parse_args(['--foo', '42']    )
        self.assertEqual(args.foo    ,      4_2    )
        lowerCAmelCase     =			parser.parse_args_into_dataclasses(['--foo', '42']    )[0]
        self.assertEqual(enum_ex.foo    ,      MixedTypeEnum.fourtytwo    )





     def 							__UpperCAmelCase      ( self       :      int    )				->  Dict:

        @dataclass
        class 		UpperCAmelCase_      :
             lowerCamelCase				:	Literal["titi", "toto", 42]		=     "toto"




        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument(
            '--foo'    ,      default='toto'    ,      choices=('titi', 'toto', 4_2)    ,      type=make_choice_type_function(['titi', 'toto', 4_2]    )    ,      )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

        lowerCAmelCase     =			parser.parse_args([]    )
        self.assertEqual(args.foo    ,      'toto'    )

        lowerCAmelCase     =			parser.parse_args(['--foo', 'titi']    )
        self.assertEqual(args.foo    ,      'titi'    )

        lowerCAmelCase     =			parser.parse_args(['--foo', '42']    )
        self.assertEqual(args.foo    ,      4_2    )





     def 							__UpperCAmelCase      ( self       :      Union[str, Any]    )				->  List[Any]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo_int'    ,      nargs='+'    ,      default=[]    ,      type=UpperCAmelCase__    )
        expected.add_argument('--bar_int'    ,      nargs='+'    ,      default=[1, 2, 3]    ,      type=UpperCAmelCase__    )
        expected.add_argument('--foo_str'    ,      nargs='+'    ,      default=['Hallo', 'Bonjour', 'Hello']    ,      type=UpperCAmelCase__    )
        expected.add_argument('--foo_float'    ,      nargs='+'    ,      default=[0.1, 0.2, 0.3]    ,      type=UpperCAmelCase__    )

        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

        lowerCAmelCase     =			parser.parse_args([]    )
        self.assertEqual(
            UpperCAmelCase__    ,      Namespace(foo_int=[]    ,      bar_int=[1, 2, 3]    ,      foo_str=['Hallo', 'Bonjour', 'Hello']    ,      foo_float=[0.1, 0.2, 0.3]    )    ,      )

        lowerCAmelCase     =			parser.parse_args('--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'.split()    )
        self.assertEqual(UpperCAmelCase__    ,      Namespace(foo_int=[1]    ,      bar_int=[2, 3]    ,      foo_str=['a', 'b', 'c']    ,      foo_float=[0.1, 0.7]    )    )





     def 							__UpperCAmelCase      ( self       :      Tuple    )				->  Union[str, Any]:
        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo'    ,      default=UpperCAmelCase__    ,      type=UpperCAmelCase__    )
        expected.add_argument('--bar'    ,      default=UpperCAmelCase__    ,      type=UpperCAmelCase__    ,      help='help message'    )
        expected.add_argument('--baz'    ,      default=UpperCAmelCase__    ,      type=UpperCAmelCase__    )
        expected.add_argument('--ces'    ,      nargs='+'    ,      default=[]    ,      type=UpperCAmelCase__    )
        expected.add_argument('--des'    ,      nargs='+'    ,      default=[]    ,      type=UpperCAmelCase__    )

        lowerCAmelCase     =			[OptionalExample]
        if is_python_no_less_than_3_10:
           dataclass_types.append(UpperCAmelCase__    )

        for dataclass_type in dataclass_types:
           lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

           self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )

           lowerCAmelCase     =			parser.parse_args([]    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=UpperCAmelCase__    ,      bar=UpperCAmelCase__    ,      baz=UpperCAmelCase__    ,      ces=[]    ,      des=[]    )    )

           lowerCAmelCase     =			parser.parse_args('--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'.split()    )
           self.assertEqual(UpperCAmelCase__    ,      Namespace(foo=1_2    ,      bar=3.14    ,      baz='42'    ,      ces=['a', 'b', 'c']    ,      des=[1, 2, 3]    )    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  List[str]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--required_list'    ,      nargs='+'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument('--required_str'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument(
            '--required_enum'    ,      type=make_choice_type_function(['titi', 'toto']    )    ,      choices=['titi', 'toto']    ,      required=UpperCAmelCase__    ,      )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Optional[Any]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			argparse.ArgumentParser()
        expected.add_argument('--foo'    ,      type=UpperCAmelCase__    ,      required=UpperCAmelCase__    )
        expected.add_argument(
            '--required_enum'    ,      type=make_choice_type_function(['titi', 'toto']    )    ,      choices=['titi', 'toto']    ,      required=UpperCAmelCase__    ,      )
        expected.add_argument('--opt'    ,      type=UpperCAmelCase__    ,      default=UpperCAmelCase__    )
        expected.add_argument('--baz'    ,      default='toto'    ,      type=UpperCAmelCase__    ,      help='help message'    )
        expected.add_argument('--foo_str'    ,      nargs='+'    ,      default=['Hallo', 'Bonjour', 'Hello']    ,      type=UpperCAmelCase__    )
        self.argparsersEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Union[str, Any]    )				->  Any:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			{
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }

        lowerCAmelCase     =			parser.parse_dict(UpperCAmelCase__    )[0]
        lowerCAmelCase     =			BasicExample(**UpperCAmelCase__    )
        self.assertEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      List[Any]    )				->  List[Any]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			{
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
            'extra': 4_2,
        }

        self.assertRaises(UpperCAmelCase__    ,      parser.parse_dict    ,      UpperCAmelCase__    ,      allow_extra_keys=UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      int    )				->  List[str]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			{
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }
        with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     =			os.path.join(UpperCAmelCase__    ,      'temp_json'    )
           os.mkdir(UpperCAmelCase__    )
           with open(temp_local_path + '.json'    ,      'w+'    ) as f:
              json.dump(UpperCAmelCase__    ,      UpperCAmelCase__    )
           lowerCAmelCase     =			parser.parse_yaml_file(Path(temp_local_path + '.json'    )    )[0]

        lowerCAmelCase     =			BasicExample(**UpperCAmelCase__    )
        self.assertEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Optional[int]    )				->  List[Any]:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )

        lowerCAmelCase     =			{
            'foo': 1_2,
            'bar': 3.14,
            'baz': '42',
            'flag': True,
        }
        with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     =			os.path.join(UpperCAmelCase__    ,      'temp_yaml'    )
           os.mkdir(UpperCAmelCase__    )
           with open(temp_local_path + '.yaml'    ,      'w+'    ) as f:
              yaml.dump(UpperCAmelCase__    ,      UpperCAmelCase__    )
           lowerCAmelCase     =			parser.parse_yaml_file(Path(temp_local_path + '.yaml'    )    )[0]
        lowerCAmelCase     =			BasicExample(**UpperCAmelCase__    )
        self.assertEqual(UpperCAmelCase__    ,      UpperCAmelCase__    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  int:
        lowerCAmelCase     =			HfArgumentParser(UpperCAmelCase__    )
        self.assertIsNotNone(UpperCAmelCase__    )









'''simple docstring'''




import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser





def 	a_					( lowerCamelCase	:     Dict				):
   lowerCAmelCase     =			{}
   lowerCAmelCase     =			tokenizer(example['content']	, truncation=lowerCamelCase				)['input_ids']
   lowerCAmelCase     =			len(example['content']				) / len(output['input_ids']				)
   return output


__snake_case    						=HfArgumentParser(PretokenizationArguments)
__snake_case    						=parser.parse_args()
if args.num_workers is None:
       __snake_case    						=multiprocessing.cpu_count()
__snake_case    						=AutoTokenizer.from_pretrained(args.tokenizer_dir)

__snake_case    						=time.time()
__snake_case    						=load_dataset(args.dataset_name, split="""train""")
print(F'''Dataset loaded in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
__snake_case    						=ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F'''Dataset tokenized in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
ds.push_to_hub(args.tokenized_data_repo)




print(F'''Data pushed to the hub in {time.time()-t_start:.2f}s''')





