






import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


__snake_case     :     Union[str, Any]       =  logging.getLogger(__name__)







def 						_UpperCamelCase (     )	-> Dict:

						"""simple docstring"""



						lowerCAmelCase__         =    argparse.ArgumentParser(description='Train a unigram tokenizer on the wikitext dataset.'		)
						parser.add_argument(
						    '--dataset_name'     ,							type=__UpperCamelCase     ,							default='wikitext'     ,							help='Name of the training. Explore datasets at: hf.co/datasets.'     ,							)
						parser.add_argument(
						    '--dataset_config'     ,							type=__UpperCamelCase     ,							default='wikitext-103-raw-v1'     ,							help='Configuration name of the dataset.'		)
						parser.add_argument(
						    '--batch_size'     ,							type=__UpperCamelCase     ,							default=1000     ,							help='Batch size during training.'     ,							)
						parser.add_argument(
						    '--vocab_size'     ,							type=__UpperCamelCase     ,							default=1_0048     ,							help='Size of the desired vocabulary.'     ,							)
						parser.add_argument(
						    '--limit'     ,							default=__UpperCamelCase     ,							type=__UpperCamelCase     ,							help='Limit the number of shards (used for debugging).'     ,							)
						parser.add_argument(
						    '--export_to_hub'     ,							action='store_true'     ,							)

						lowerCAmelCase__         =    parser.parse_args()
						return args







def 						_UpperCamelCase (     UpperCamelCase_							:       int		)	-> List[str]:

						"""simple docstring"""



						lowerCAmelCase__         =    datasets.load_dataset(args.dataset_name     ,							args.dataset_config     ,							split='train'		)

						if args.limit is not None:
												lowerCAmelCase__         =    min(len(__UpperCamelCase		)     ,							args.limit		)
												lowerCAmelCase__         =    dataset.select(range(__UpperCamelCase		)		)
												logger.info(F"Limiting the dataset to {args.limit} entries."		)

						def batch_iterator():
												for i in range(0     ,							len(__UpperCamelCase		)     ,							args.batch_size		):
																		yield dataset[i : i + args.batch_size]["text"]

												# Prepare the tokenizer.
						lowerCAmelCase__         =    Tokenizer(Unigram()		)
						lowerCAmelCase__         =    normalizers.Sequence([normalizers.Replace('``'     ,							'\"'		), normalizers.Replace('\'\''     ,							'\"'		)]		)
						lowerCAmelCase__         =    pre_tokenizers.Metaspace()

						# Prepare the trainer.
						lowerCAmelCase__         =    UnigramTrainer(
						    unk_token='<unk>'     ,							special_tokens=['[CLS]', '[SEP]', '<unk>', '<pad>', '[MASK]']     ,							vocab_size=args.vocab_size     ,							)

						logger.info('Training the tokenizer.'		)
						tokenizer.train_from_iterator(batch_iterator()     ,							trainer=__UpperCamelCase		)
						logger.info('Tokenizer training complete!'		)

						lowerCAmelCase__         =    tokenizer.token_to_id('[CLS]'		)
						lowerCAmelCase__         =    tokenizer.token_to_id('[SEP]'		)
						lowerCAmelCase__         =    processors.TemplateProcessing(
						    single='[CLS]:0 $A:0 [SEP]:0'     ,							pair='[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1'     ,							special_tokens=[
						        ('[CLS]', cls_token_id),
						        ('[SEP]', sep_token_id),
						    ]     ,							)
						lowerCAmelCase__         =    decoders.Metaspace()

						if args.export_to_hub:
												logger.info('Exporting the trained tokenzier to Hub.'		)
												lowerCAmelCase__         =    AlbertTokenizerFast(tokenizer_object=__UpperCamelCase		)
												new_tokenizer.push_to_hub('unigram-tokenizer-dataset'		)


if __name__ == "__main__":
					__snake_case     :     str       =  parse_args()
					main(args)






import hashlib
import importlib.util
import json
import os
import pathlib
from types import ModuleType

import pytest
import requests

__lowerCamelCase     :					Optional[Any]				  =						pathlib.Path.cwd().joinpath('''project_euler''')
__lowerCamelCase     :					Dict				  =						pathlib.Path.cwd().joinpath(
    '''scripts''', '''project_euler_answers.json'''
)

with open(PROJECT_EULER_ANSWERS_PATH) as file_handle:
   __lowerCamelCase     :					dict[str, str]				  =						json.load(file_handle)



def       __SCREAMING_SNAKE_CASE    (   __UpperCamelCase       :			pathlib.Path      )				->   ModuleType:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__     				= importlib.util.spec_from_file_location(file_path.name					,	str(__UpperCamelCase      )      )
  SCREAMING_SNAKE_CASE__     				= importlib.util.module_from_spec(__UpperCamelCase      )  # type: ignore
  spec.loader.exec_module(__UpperCamelCase      )  # type: ignore
  return module



def       __SCREAMING_SNAKE_CASE    (   )				->   list[pathlib.Path]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__     				= []
  for problem_dir_path in PROJECT_EULER_DIR_PATH.iterdir():
    if problem_dir_path.is_file() or problem_dir_path.name.startswith("""_"""      ):
      continue
    for file_path in problem_dir_path.iterdir():
      if file_path.suffix != ".py" or file_path.name.startswith(("""_""", """test""")      ):
        continue
      solution_file_paths.append(__UpperCamelCase      )
  return solution_file_paths



def       __SCREAMING_SNAKE_CASE    (   )				->   str:







  """simple docstring"""





  with open(os.environ["""GITHUB_EVENT_PATH"""]      ) as file:
    SCREAMING_SNAKE_CASE__     				= json.load(__UpperCamelCase      )
  return event["pull_request"]["url"] + "/files"



def       __SCREAMING_SNAKE_CASE    (   )				->   list[pathlib.Path]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__     				= []
  SCREAMING_SNAKE_CASE__     				= {
      """Accept""": """application/vnd.github.v3+json""",
      """Authorization""": """token """ + os.environ["""GITHUB_TOKEN"""],
  }
  SCREAMING_SNAKE_CASE__     				= requests.get(get_files_url()					,	headers=__UpperCamelCase      ).json()
  for file in files:
    SCREAMING_SNAKE_CASE__     				= pathlib.Path.cwd().joinpath(file["""filename"""]      )
    if (
        filepath.suffix != ".py"
        or filepath.name.startswith(("""_""", """test""")      )
        or not filepath.name.startswith("""sol"""      )
    ):
      continue
    solution_file_paths.append(__UpperCamelCase      )
  return solution_file_paths



def       __SCREAMING_SNAKE_CASE    (   )				->   list[pathlib.Path]:







  """simple docstring"""





  if os.environ.get("""CI"""      ) and os.environ.get("""GITHUB_EVENT_NAME"""      ) == "pull_request":
    # Return only if there are any, otherwise default to all solutions
    if filepaths := added_solution_file_path():
      return filepaths
  return all_solution_file_paths()






@pytest.mark.parametrize(
    """solution_path"""					,	collect_solution_file_paths()					,	ids=lambda __UpperCamelCase      : f"""{path.parent.name}/{path.name}"""					,	)
def       __SCREAMING_SNAKE_CASE    (   __UpperCamelCase       :			pathlib.Path      )				->   None:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__     				= solution_path.parent.name[8:].zfill(3      )
  SCREAMING_SNAKE_CASE__     				= PROBLEM_ANSWERS[problem_number]
  SCREAMING_SNAKE_CASE__     				= convert_path_to_module(__UpperCamelCase      )
  SCREAMING_SNAKE_CASE__     				= str(solution_module.solution()      )  # type: ignore
  SCREAMING_SNAKE_CASE__     				= hashlib.shaaaa(answer.encode()      ).hexdigest()
  assert (
      answer == expected
  ), f"""Expected solution to {problem_number} to have hash {expected}, got {answer}"""
