




'''simple docstring'''

import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


a  :				Dict									=  re.compile(r'\s+')


def 		__magic_name__    (	__UpperCAmelCase	)		->					Tuple:

					'''simple docstring'''

					return {"hash": hashlib.mda(re.sub(__UpperCAmelCase,      '''''',      example['''content''']	).encode('''utf-8'''	)	).hexdigest()}


def 		__magic_name__    (	__UpperCAmelCase	)		->					Any:

					'''simple docstring'''

					snake_case_									=				[len(__UpperCAmelCase	) for line in example['''content'''].splitlines()]
					return {"line_mean": np.mean(__UpperCAmelCase	), "line_max": max(__UpperCAmelCase	)}


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					snake_case_									=				np.mean([c.isalnum() for c in example['''content''']]	)
					return {"alpha_frac": alpha_frac}


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase	)		->					List[Any]:

					'''simple docstring'''

					if example["hash"] in uniques:
										uniques.remove(example['''hash''']	)
										return True
					else:
										return False


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase=5	)		->					Dict:

					'''simple docstring'''

					snake_case_									=				['''auto-generated''', '''autogenerated''', '''automatically generated''']
					snake_case_									=				example['''content'''].splitlines()
					for _, line in zip(range(__UpperCAmelCase	),      __UpperCAmelCase	):
										for keyword in keywords:
															if keyword in line.lower():
																				return {"autogenerated": True}
					else:
										return {"autogenerated": False}


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase=5,      __UpperCAmelCase=0.0_5	)		->					Optional[int]:

					'''simple docstring'''

					snake_case_									=				['''unit tests''', '''test file''', '''configuration file''']
					snake_case_									=				example['''content'''].splitlines()
					snake_case_									=				0
					snake_case_									=				0
					# first test
					for _, line in zip(range(__UpperCAmelCase	),      __UpperCAmelCase	):
										for keyword in keywords:
															if keyword in line.lower():
																				return {"config_or_test": True}
    # second test
					snake_case_									=				example['''content'''].count('''\n'''	)
					snake_case_									=				int(coeff * nlines	)
					for line in lines:
										count_config += line.lower().count('''config'''	)
										count_test += line.lower().count('''test'''	)
										if count_config > threshold or count_test > threshold:
															return {"config_or_test": True}
					return {"config_or_test": False}


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					snake_case_									=				['''def ''', '''class ''', '''for ''', '''while ''']
					snake_case_									=				example['''content'''].splitlines()
					for line in lines:
										for keyword in keywords:
															if keyword in line.lower():
																				return {"has_no_keywords": False}
					return {"has_no_keywords": True}


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase=4	)		->					Tuple:

					'''simple docstring'''

					snake_case_									=				example['''content'''].splitlines()
					snake_case_									=				0
					for line in lines:
										counter += line.lower().count('''='''	)
										if counter > minimum:
															return {"has_few_assignments": False}
					return {"has_few_assignments": True}


def 		__magic_name__    (	__UpperCAmelCase	)		->					str:

					'''simple docstring'''

					snake_case_									=				tokenizer(example['''content'''],      truncation=__UpperCAmelCase	)['''input_ids''']
					snake_case_									=				len(example['''content''']	) / len(__UpperCAmelCase	)
					return {"ratio": ratio}


def 		__magic_name__    (	__UpperCAmelCase	)		->					Any:

					'''simple docstring'''

					snake_case_									=				{}
					results.update(get_hash(__UpperCAmelCase	)	)
					results.update(line_stats(__UpperCAmelCase	)	)
					results.update(alpha_stats(__UpperCAmelCase	)	)
					results.update(char_token_ratio(__UpperCAmelCase	)	)
					results.update(is_autogenerated(__UpperCAmelCase	)	)
					results.update(is_config_or_test(__UpperCAmelCase	)	)
					results.update(has_no_keywords(__UpperCAmelCase	)	)
					results.update(has_few_assignments(__UpperCAmelCase	)	)
					return results


def 		__magic_name__    (	__UpperCAmelCase,      __UpperCAmelCase,      __UpperCAmelCase	)		->					Any:

					'''simple docstring'''

					if not check_uniques(__UpperCAmelCase,      __UpperCAmelCase	):
										return False
					elif example["autogenerated"]:
										return False
					elif example["line_max"] > args.line_max:
										return False
					elif example["line_mean"] > args.line_mean:
										return False
					elif example["alpha_frac"] < args.alpha_frac:
										return False
					elif example["ratio"] < args.min_token_ratio:
										return False
					elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
										return False
					elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
										return False
					elif example["has_few_assignments"]:
										return False
					else:
										return True


def 		__magic_name__    (	__UpperCAmelCase	)		->					Optional[int]:

					'''simple docstring'''

					with open(__UpperCAmelCase,      '''rb'''	) as f_in:
										with gzip.open(str(__UpperCAmelCase	) + '''.gz''',      '''wb''',      compresslevel=6	) as f_out:
															shutil.copyfileobj(__UpperCAmelCase,      __UpperCAmelCase	)
					os.unlink(__UpperCAmelCase	)


# Settings
a  :				List[Any]									=  HfArgumentParser(PreprocessingArguments)
a  :				Any									=  parser.parse_args()
if args.num_workers is None:
							a  :				Union[str, Any]									=  multiprocessing.cpu_count()
a  :				List[str]									=  AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
a  :				Union[str, Any]									=  time.time()
a  :				int									=  load_dataset(args.dataset_name, split='train')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
a  :				int									=  time.time()
a  :				List[str]									=  ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
a  :				str									=  set(ds.unique('hash'))
a  :				List[Any]									=  len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
a  :				List[str]									=  time.time()
a  :				Tuple									=  ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
							a  :				Tuple									=  time.time()
							a		,a  :				Optional[int]									=  deduplicate_dataset(ds_filter, args.jaccard_threshold)
							print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
							print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
a  :				Optional[int]									=  Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
							with open(output_dir / 'duplicate_clusters.json', 'w') as f:
														json.dump(duplicate_clusters, f)

a  :				Dict									=  output_dir / 'data'
data_dir.mkdir(exist_ok=True)

a  :				Optional[int]									=  time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
							a  :				str									=  str(data_dir / f'''file-{file_number+1:012}.json''')
							a  :				int									=  min(len(ds_filter), index + args.samples_per_file)
							ds_filter.select(list(range(index, end_index))).to_json(file_path)
							compress_file(file_path)

print(f'''Time to save dataset: {time.time()-t_start:.2f}''')







'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a  :				Dict									=  logging.get_logger(__name__)

a  :				List[str]									=  {
    'Helsinki-NLP/opus-mt-en-de': 'https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/config.json',
    # See all Marian models at https://huggingface.co/models?filter=marian
}






class 						a				(   _lowerCamelCase     ):
				snake_case_      =		"marian"
				snake_case_      =		["past_key_values"]
				snake_case_      =		{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}


				def __init__(		self    :					List[Any]       ,				lowercase_    :					Optional[Any]=5_8101       ,				lowercase_    :					Dict=None       ,				lowercase_    :					List[str]=1024       ,				lowercase_    :					Optional[Any]=12       ,				lowercase_    :					int=4096       ,				lowercase_    :					Any=16       ,				lowercase_    :					Optional[int]=12       ,				lowercase_    :					str=4096       ,				lowercase_    :					Union[str, Any]=16       ,				lowercase_    :					Dict=0.0       ,				lowercase_    :					Union[str, Any]=0.0       ,				lowercase_    :					Optional[Any]=True       ,				lowercase_    :					Union[str, Any]=True       ,				lowercase_    :					int="gelu"       ,				lowercase_    :					Dict=1024       ,				lowercase_    :					int=0.1       ,				lowercase_    :					Tuple=0.0       ,				lowercase_    :					Tuple=0.0       ,				lowercase_    :					Tuple=0.02       ,				lowercase_    :					int=5_8100       ,				lowercase_    :					Optional[Any]=False       ,				lowercase_    :					Any=5_8100       ,				lowercase_    :					Optional[int]=0       ,				lowercase_    :					Dict=0       ,				lowercase_    :					List[str]=True       ,				**lowercase_    :					Any       ,				):
									snake_case_									=				vocab_size
									snake_case_									=				decoder_vocab_size or vocab_size
									snake_case_									=				max_position_embeddings
									snake_case_									=				d_model
									snake_case_									=				encoder_ffn_dim
									snake_case_									=				encoder_layers
									snake_case_									=				encoder_attention_heads
									snake_case_									=				decoder_ffn_dim
									snake_case_									=				decoder_layers
									snake_case_									=				decoder_attention_heads
									snake_case_									=				dropout
									snake_case_									=				attention_dropout
									snake_case_									=				activation_dropout
									snake_case_									=				activation_function
									snake_case_									=				init_std
									snake_case_									=				encoder_layerdrop
									snake_case_									=				decoder_layerdrop
									snake_case_									=				use_cache
									snake_case_									=				encoder_layers
									snake_case_									=				scale_embedding  # scale factor will be sqrt(d_model) if True
									snake_case_									=				share_encoder_decoder_embeddings
									super().__init__(
									    pad_token_id=lowercase_       ,				eos_token_id=lowercase_       ,				is_encoder_decoder=lowercase_       ,				decoder_start_token_id=lowercase_       ,				forced_eos_token_id=lowercase_       ,				**lowercase_       ,				)







class 						a				(   _lowerCamelCase     ):


				@property
				# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.inputs
				def   A_ (		self    :					Union[str, Any]					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														    ]					)

														if self.use_past:
																			snake_case_									=				{0: '''batch'''}
																			snake_case_									=				{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
														else:
																			snake_case_									=				{0: '''batch''', 1: '''decoder_sequence'''}
																			snake_case_									=				{0: '''batch''', 1: '''decoder_sequence'''}

														if self.use_past:
																			self.fill_with_past_key_values_(lowercase_       ,				direction='''inputs'''					)
									elif self.task == "causal-lm":
														# TODO: figure this case out.
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														    ]					)
														if self.use_past:
																			snake_case_			,snake_case_									=				self.num_layers
																			for i in range(lowercase_					):
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
									else:
														snake_case_									=				OrderedDict(
														    [
														        ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
														        ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
														        ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
														    ]					)

									return common_inputs


				@property
				# Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
				def   A_ (		self    :					Dict					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				super().outputs
									else:
														snake_case_									=				super(lowercase_       ,				self					).outputs
														if self.use_past:
																			snake_case_			,snake_case_									=				self.num_layers
																			for i in range(lowercase_					):
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
																								snake_case_									=				{0: '''batch''', 2: '''past_sequence + sequence'''}
									return common_outputs


				def   A_ (		self    :					Dict       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)

									# Generate decoder inputs
									snake_case_									=				seq_length if not self.use_past else 1
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)
									snake_case_									=				{F"decoder_{name}": tensor for name, tensor in decoder_inputs.items()}
									snake_case_									=				dict(**lowercase_       ,				**lowercase_					)

									if self.use_past:
														if not is_torch_available():
																			raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
														else:
																			import torch
														snake_case_			,snake_case_									=				common_inputs['''input_ids'''].shape
														snake_case_									=				common_inputs['''decoder_input_ids'''].shape[1]
														snake_case_			,snake_case_									=				self.num_attention_heads
														snake_case_									=				(
														    batch,
														    num_encoder_attention_heads,
														    encoder_seq_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)
														snake_case_									=				decoder_seq_length + 3
														snake_case_									=				(
														    batch,
														    num_decoder_attention_heads,
														    decoder_past_length,
														    self._config.hidden_size // num_decoder_attention_heads,
														)

														snake_case_									=				torch.cat(
														    [common_inputs['''decoder_attention_mask'''], torch.ones(lowercase_       ,				lowercase_					)]       ,				dim=1					)

														snake_case_									=				[]
														# If the number of encoder and decoder layers are present in the model configuration, both are considered
														snake_case_			,snake_case_									=				self.num_layers
														snake_case_									=				min(lowercase_       ,				lowercase_					)
														snake_case_									=				max(lowercase_       ,				lowercase_					) - min_num_layers
														snake_case_									=				'''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

														for _ in range(lowercase_					):
																			common_inputs["past_key_values"].append(
																			    (
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			        torch.zeros(lowercase_					),
																			    )					)
														# TODO: test this.
														snake_case_									=				encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
														for _ in range(lowercase_       ,				lowercase_					):
																			common_inputs["past_key_values"].append((torch.zeros(lowercase_					), torch.zeros(lowercase_					))					)
									return common_inputs


				def   A_ (		self    :					Union[str, Any]       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									snake_case_									=				self._generate_dummy_inputs_for_encoder_and_decoder(
									    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)

									if self.use_past:
														if not is_torch_available():
																			raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
														else:
																			import torch
														snake_case_			,snake_case_									=				common_inputs['''input_ids'''].shape
														# Not using the same length for past_key_values
														snake_case_									=				seqlen + 2
														snake_case_			,snake_case_									=				self.num_layers
														snake_case_			,snake_case_									=				self.num_attention_heads
														snake_case_									=				(
														    batch,
														    num_encoder_attention_heads,
														    past_key_values_length,
														    self._config.hidden_size // num_encoder_attention_heads,
														)

														snake_case_									=				common_inputs['''attention_mask'''].dtype
														snake_case_									=				torch.cat(
														    [common_inputs['''attention_mask'''], torch.ones(lowercase_       ,				lowercase_       ,				dtype=lowercase_					)]       ,				dim=1					)
														snake_case_									=				[
														    (torch.zeros(lowercase_					), torch.zeros(lowercase_					)) for _ in range(lowercase_					)
														]
									return common_inputs


				def   A_ (		self    :					List[str]       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									# Copied from OnnxConfig.generate_dummy_inputs
									# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
									# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
									snake_case_									=				compute_effective_axis_dimension(
									    lowercase_       ,				fixed_dimension=OnnxConfig.default_fixed_batch       ,				num_token_to_add=0					)

									# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
									snake_case_									=				tokenizer.num_special_tokens_to_add(lowercase_					)
									snake_case_									=				compute_effective_axis_dimension(
									    lowercase_       ,				fixed_dimension=OnnxConfig.default_fixed_sequence       ,				num_token_to_add=lowercase_					)

									# Generate dummy inputs according to compute batch and sequence
									snake_case_									=				[''' '''.join([tokenizer.unk_token]					) * seq_length] * batch_size
									snake_case_									=				dict(tokenizer(lowercase_       ,				return_tensors=lowercase_					)					)
									return common_inputs


				def   A_ (		self    :					Any       ,				lowercase_    :					PreTrainedTokenizer       ,				lowercase_    :					int = -1       ,				lowercase_    :					int = -1       ,				lowercase_    :					bool = False       ,				lowercase_    :					Optional[TensorType] = None       ,				):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				self._generate_dummy_inputs_for_default_and_seqaseq_lm(
														    lowercase_       ,				batch_size=lowercase_       ,				seq_length=lowercase_       ,				is_pair=lowercase_       ,				framework=lowercase_					)

									else:
														snake_case_									=				self._generate_dummy_inputs_for_causal_lm(
														    lowercase_       ,				batch_size=lowercase_       ,				seq_length=lowercase_       ,				is_pair=lowercase_       ,				framework=lowercase_					)

									return common_inputs


				def   A_ (		self    :					Dict       ,				lowercase_    :					List[str]       ,				lowercase_    :					List[str]       ,				lowercase_    :					int       ,				lowercase_    :					List[str]					):
									if self.task in ["default", "seq2seq-lm"]:
														snake_case_									=				super()._flatten_past_key_values_(lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)
									else:
														snake_case_									=				super(lowercase_       ,				self					)._flatten_past_key_values_(
														    lowercase_       ,				lowercase_       ,				lowercase_       ,				lowercase_					)


				@property
				def   A_ (		self    :					List[str]					):
									return 1e-4

