


import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowerCAmelCase__  :int   			=							logging.getLogger(__name__)

lowerCAmelCase__  :Union[str, Any]   			=							tf.data.AUTOTUNE

def        lowerCAmelCase__ (			)  ->		Optional[int]:


							'''simple docstring'''





							_UpperCAmelCase 		=			argparse.ArgumentParser(description='Train a masked language model on TPU.'       )
							parser.add_argument(
							    '--pretrained_model_config'			,  type=lowerCAmelCase__			,  default='roberta-base'			,  help='The model config to use. Note that we don\'t copy the model\'s weights, only the config!'			,  )
							parser.add_argument(
							    '--tokenizer'			,  type=lowerCAmelCase__			,  default='unigram-tokenizer-wikitext'			,  help='The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'			,  )

							parser.add_argument(
							    '--per_replica_batch_size'			,  type=lowerCAmelCase__			,  default=8			,  help='Batch size per TPU core.'			,  )

							parser.add_argument(
							    '--no_tpu'			,  action='store_true'			,  help='If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'			,  )

							parser.add_argument(
							    '--tpu_name'			,  type=lowerCAmelCase__			,  help='Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'			,  default='local'			,  )

							parser.add_argument(
							    '--tpu_zone'			,  type=lowerCAmelCase__			,  help='Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'			,  )

							parser.add_argument(
							    '--gcp_project'			,  type=lowerCAmelCase__			,  help='Google cloud project name. Only used for non-Colab TPU nodes.'       )

							parser.add_argument(
							    '--bfloat16'			,  action='store_true'			,  help='Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'			,  )

							parser.add_argument(
							    '--train_dataset'			,  type=lowerCAmelCase__			,  help='Path to training dataset to load. If the path begins with `gs://`'
							    ' then the dataset will be loaded from a Google Cloud Storage bucket.'			,  )

							parser.add_argument(
							    '--shuffle_buffer_size'			,  type=lowerCAmelCase__			,  default=2**1_8			,  help='Size of the shuffle buffer (in samples)'			,  )

							parser.add_argument(
							    '--eval_dataset'			,  type=lowerCAmelCase__			,  help='Path to evaluation dataset to load. If the path begins with `gs://`'
							    ' then the dataset will be loaded from a Google Cloud Storage bucket.'			,  )

							parser.add_argument(
							    '--num_epochs'			,  type=lowerCAmelCase__			,  default=1			,  help='Number of epochs to train for.'			,  )

							parser.add_argument(
							    '--learning_rate'			,  type=lowerCAmelCase__			,  default=1e-4			,  help='Learning rate to use for training.'			,  )

							parser.add_argument(
							    '--weight_decay_rate'			,  type=lowerCAmelCase__			,  default=1e-3			,  help='Weight decay rate to use for training.'			,  )

							parser.add_argument(
							    '--max_length'			,  type=lowerCAmelCase__			,  default=5_1_2			,  help='Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'			,  )

							parser.add_argument(
							    '--mlm_probability'			,  type=lowerCAmelCase__			,  default=0.15			,  help='Fraction of tokens to mask during training.'			,  )

							parser.add_argument('--output_dir'			,  type=lowerCAmelCase__			,  required=lowerCAmelCase__			,  help='Path to save model checkpoints to.'       )
							parser.add_argument('--hub_model_id'			,  type=lowerCAmelCase__			,  help='Model ID to upload to on the Hugging Face Hub.'       )

							_UpperCAmelCase 		=			parser.parse_args()
							return args

def        lowerCAmelCase__ (			a__:		Optional[int]       )  ->		int:


							'''simple docstring'''





							try:
														if args.tpu_name:
																					_UpperCAmelCase 		=			tf.distribute.cluster_resolver.TPUClusterResolver(
																					    args.tpu_name			,  zone=args.tpu_zone			,  project=args.gcp_project       )
														else:
																					_UpperCAmelCase 		=			tf.distribute.cluster_resolver.TPUClusterResolver()
							except ValueError:
														raise RuntimeError(
														    'Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '
														    '--gcp_project. When running on a TPU VM, use --tpu_name local.'       )

							tf.config.experimental_connect_to_cluster(lowerCAmelCase__       )
							tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase__       )

							return tpu

def        lowerCAmelCase__ (			a__:		Dict       )  ->		Dict:


							'''simple docstring'''





							_UpperCAmelCase 		=			0
							for file in file_list:
														_UpperCAmelCase 		=			file.split('/'       )[-1]
														_UpperCAmelCase 		=			re.search(R'-\d+-(\d+)\.tfrecord'			,  lowerCAmelCase__       ).group(1       )
														_UpperCAmelCase 		=			int(lowerCAmelCase__       )
														num_samples += sample_count

							return num_samples

def        lowerCAmelCase__ (			a__:		str			,  a__:		Optional[int]			,  a__:		Tuple			,  a__:		Any			,  a__:		Dict			,  a__:		Optional[int]=None       )  ->		Tuple:


							'''simple docstring'''





							_UpperCAmelCase 		=			count_samples(lowerCAmelCase__       )
							_UpperCAmelCase 		=			tf.data.Dataset.from_tensor_slices(lowerCAmelCase__       )
							if shuffle:
														_UpperCAmelCase 		=			dataset.shuffle(len(lowerCAmelCase__       )       )
							_UpperCAmelCase 		=			tf.data.TFRecordDataset(lowerCAmelCase__			,  num_parallel_reads=lowerCAmelCase__       )
							# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
							_UpperCAmelCase 		=			dataset.apply(tf.data.experimental.assert_cardinality(lowerCAmelCase__       )       )
							_UpperCAmelCase 		=			dataset.map(lowerCAmelCase__			,  num_parallel_calls=lowerCAmelCase__       )
							if shuffle:
														assert shuffle_buffer_size is not None
														_UpperCAmelCase 		=			dataset.shuffle(args.shuffle_buffer_size       )
							_UpperCAmelCase 		=			dataset.batch(lowerCAmelCase__			,  drop_remainder=lowerCAmelCase__       )
							_UpperCAmelCase 		=			dataset.map(lowerCAmelCase__			,  num_parallel_calls=lowerCAmelCase__       )
							_UpperCAmelCase 		=			dataset.prefetch(lowerCAmelCase__       )
							return dataset

def        lowerCAmelCase__ (			a__:		str       )  ->		List[str]:


							'''simple docstring'''





							if not args.no_tpu:
														_UpperCAmelCase 		=			initialize_tpu(lowerCAmelCase__       )
														_UpperCAmelCase 		=			tf.distribute.TPUStrategy(lowerCAmelCase__       )
							else:
														_UpperCAmelCase 		=			tf.distribute.OneDeviceStrategy(device='/gpu:0'       )

							if args.bfloataa:
														tf.keras.mixed_precision.set_global_policy('mixed_bfloat16'       )

							_UpperCAmelCase 		=			AutoTokenizer.from_pretrained(args.tokenizer       )
							_UpperCAmelCase 		=			AutoConfig.from_pretrained(args.pretrained_model_config       )
							_UpperCAmelCase 		=			tokenizer.vocab_size

							_UpperCAmelCase 		=			tf.io.gfile.glob(os.path.join(args.train_dataset			,  '*.tfrecord'       )       )
							if not training_records:
														raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.'''       )
							_UpperCAmelCase 		=			tf.io.gfile.glob(os.path.join(args.eval_dataset			,  '*.tfrecord'       )       )
							if not eval_records:
														raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.'''       )

							_UpperCAmelCase 		=			count_samples(lowerCAmelCase__       )

							_UpperCAmelCase 		=			num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
							_UpperCAmelCase 		=			steps_per_epoch * args.num_epochs

							with strategy.scope():
														_UpperCAmelCase 		=			TFAutoModelForMaskedLM.from_config(lowerCAmelCase__       )
														model(model.dummy_inputs       )  # Pass some dummy inputs through the model to ensure all the weights are built
														_UpperCAmelCase		,       _UpperCAmelCase 		=			create_optimizer(
														    num_train_steps=lowerCAmelCase__			,  num_warmup_steps=total_train_steps // 2_0			,  init_lr=args.learning_rate			,  weight_decay_rate=args.weight_decay_rate			,  )

														# Transformers models compute the right loss for their task by default when labels are passed, and will
														# use this for training unless you specify your own loss function in compile().
														model.compile(optimizer=lowerCAmelCase__			,  metrics=['accuracy']       )

							def decode_fn(a__:		Tuple       ):
														_UpperCAmelCase 		=			{
														    'input_ids': tf.io.FixedLenFeature(dtype=tf.intaa			,  shape=(args.max_length,)       ),
														    'attention_mask': tf.io.FixedLenFeature(dtype=tf.intaa			,  shape=(args.max_length,)       ),
														}
														return tf.io.parse_single_example(lowerCAmelCase__			,  lowerCAmelCase__       )

							# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
							# use their methods in our data pipeline.
							_UpperCAmelCase 		=			DataCollatorForLanguageModeling(
							    tokenizer=lowerCAmelCase__			,  mlm_probability=args.mlm_probability			,  mlm=lowerCAmelCase__			,  return_tensors='tf'       )

							def mask_with_collator(a__:		List[Any]       ):
														# TF really needs an isin() function
														_UpperCAmelCase 		=			(
														    ~tf.cast(batch['attention_mask']			,  tf.bool       )
														    | (batch['input_ids'] == tokenizer.cls_token_id)
														    | (batch['input_ids'] == tokenizer.sep_token_id)
														)
														_UpperCAmelCase		,       _UpperCAmelCase 		=			data_collator.tf_mask_tokens(
														    batch['input_ids']			,  vocab_size=len(lowerCAmelCase__       )			,  mask_token_id=tokenizer.mask_token_id			,  special_tokens_mask=lowerCAmelCase__			,  )
														return batch

							_UpperCAmelCase 		=			args.per_replica_batch_size * strategy.num_replicas_in_sync

							_UpperCAmelCase 		=			prepare_dataset(
							    lowerCAmelCase__			,  decode_fn=lowerCAmelCase__			,  mask_fn=lowerCAmelCase__			,  batch_size=lowerCAmelCase__			,  shuffle=lowerCAmelCase__			,  shuffle_buffer_size=args.shuffle_buffer_size			,  )

							_UpperCAmelCase 		=			prepare_dataset(
							    lowerCAmelCase__			,  decode_fn=lowerCAmelCase__			,  mask_fn=lowerCAmelCase__			,  batch_size=lowerCAmelCase__			,  shuffle=lowerCAmelCase__			,  )

							_UpperCAmelCase 		=			[]
							if args.hub_model_id:
														callbacks.append(
														    PushToHubCallback(output_dir=args.output_dir			,  hub_model_id=args.hub_model_id			,  tokenizer=lowerCAmelCase__       )       )

							model.fit(
							    lowerCAmelCase__			,  validation_data=lowerCAmelCase__			,  epochs=args.num_epochs			,  callbacks=lowerCAmelCase__			,  )

							model.save_pretrained(args.output_dir       )


if __name__ == "__main__":
		lowerCAmelCase__  :Any   			=							parse_args()
		main(args)




"""simple docstring"""




import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


lowercase_			     =  re.compile("[^A-Za-z_0-9]")
# parameters used in DuplicationIndex
lowercase_			     =  1_0
lowercase_			     =  2_5_6

def        lowercase							(		lowerCAmelCase__ :     List[str]			) ->      Optional[MinHash]:
		if len(lowerCAmelCase__			) < MIN_NUM_TOKENS:
				return None
		__a           =			MinHash(num_perm=lowerCAmelCase__			)
		for token in set(lowerCAmelCase__			):
				min_hash.update(token.encode()			)
		return min_hash

def        lowercase							(		lowerCAmelCase__ :     str			) ->      Set[str]:
		return {t for t in NON_ALPHA.split(lowerCAmelCase__			) if len(t.strip()			) > 0}
class 					__lowerCAmelCase					:


		'''simple docstring'''



		def __init__(  self							, *,
		    _a = 0.85							, ):
				__a           =			duplication_jaccard_threshold
				__a           =			NUM_PERM
				__a           =			MinHashLSH(threshold=self._duplication_jaccard_threshold							, num_perm=self._num_perm      )

				__a           =			defaultdict(_a      )

		def 			__UpperCAmelCase       (  self							, _a							, _a      ):
				__a           =			self._index.query(_a      )
				if code_key in self._index.keys:
						print(f'''Duplicate key {code_key}'''      )
						return

				self._index.insert(_a							, _a      )
				if len(_a      ) > 0:
						for base_duplicate in close_duplicates:
								if base_duplicate in self._duplicate_clusters:
										self._duplicate_clusters[base_duplicate].add(_a      )
										break
						else:
								self._duplicate_clusters[close_duplicates[0]].add(_a      )

		def 			__UpperCAmelCase       (  self      ):
				__a           =			[]
				for base, duplicates in self._duplicate_clusters.items():
						__a           =			[base] + list(_a      )
						# reformat the cluster to be a list of dict
						__a           =			[{'''base_index''': el[0], '''repo_name''': el[1], '''path''': el[2]} for el in cluster]
						duplicate_clusters.append(_a      )
				return duplicate_clusters





		def 			__UpperCAmelCase       (  self							, _a      ):
				__a           =			self.get_duplicate_clusters()
				with open(_a							, '''w'''      ) as f:
						json.dump(_a							, _a      )







def        lowercase							(		lowerCAmelCase__ :     List[str]			) ->      int:
		__a      ,     __a           =			element
		__a           =			get_min_hash([t for t in NON_ALPHA.split(data['''content''']			) if len(t.strip()			) > 0]			)
		if min_hash is not None:
				return (index, data["repo_name"], data["path"]), min_hash

def        lowercase							(		lowerCAmelCase__ :     Type[Dataset]			) ->      str:
		with mp.Pool() as pool:
				for data in pool.imap_unordered(
				    _compute_min_hash    ,					ThreadedIterator(lowerCAmelCase__    ,					max_queue_size=10000			)    ,					chunksize=100    ,					):
						if data is not None:
								yield data

def        lowercase							(		lowerCAmelCase__ :     Type[Dataset]    ,					lowerCAmelCase__ :     float			) ->      Dict:
		__a           =			DuplicationIndex(duplication_jaccard_threshold=lowerCAmelCase__			)

		for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(lowerCAmelCase__			)			)    ,					max_queue_size=100			)			):
				di.add(lowerCAmelCase__    ,					lowerCAmelCase__			)

		# Returns a List[Cluster] where Cluster is List[str] with the filenames.
		return di.get_duplicate_clusters()

def        lowercase							(		lowerCAmelCase__ :     str    ,					lowerCAmelCase__ :     str			) ->      float:
		__a           =			get_tokens(lowerCAmelCase__			)
		__a           =			get_tokens(lowerCAmelCase__			)
		return len(tokensa & tokensa			) / len(tokensa | tokensa			)


lowercase_			     =  None

def        lowercase							(		lowerCAmelCase__ :     Optional[Any]    ,					lowerCAmelCase__ :     Union[str, Any]			) ->      Any:
		__a           =			[]
		for elementa in cluster:
				__a           =			_shared_dataset[elementa['''base_index''']]['''content''']
				for elementa in extremes:
						__a           =			_shared_dataset[elementa['''base_index''']]['''content''']
						if jaccard_similarity(lowerCAmelCase__    ,					lowerCAmelCase__			) >= jaccard_threshold:
								elementa["copies"] += 1
								break
				else:
						__a           =			1
						extremes.append(lowerCAmelCase__			)
		return extremes

def        lowercase							(		lowerCAmelCase__ :     Union[str, Any]    ,					lowerCAmelCase__ :     Optional[Any]    ,					lowerCAmelCase__ :     Optional[int]			) ->      Optional[int]:
		global _shared_dataset
		__a           =			dataset
		__a           =			[]
		__a           =			partial(_find_cluster_extremes_shared    ,					jaccard_threshold=lowerCAmelCase__			)
		with mp.Pool() as pool:
				for extremes in tqdm(
				    pool.imap_unordered(
				        lowerCAmelCase__    ,					lowerCAmelCase__    ,					)    ,					total=len(lowerCAmelCase__			)    ,					):
						extremes_list.append(lowerCAmelCase__			)
		return extremes_list




def        lowercase							(		lowerCAmelCase__ :     Type[Dataset]    ,					lowerCAmelCase__ :     float = 0.85			) ->      Tuple[Type[Dataset], List[List[Dict]]]:
		__a           =			make_duplicate_clusters(lowerCAmelCase__    ,					lowerCAmelCase__			)
		__a           =			{x['''base_index'''] for cluster in duplicate_clusters for x in cluster}
		__a           =			{}
		__a           =			find_extremes(lowerCAmelCase__    ,					lowerCAmelCase__    ,					lowerCAmelCase__			)
		for extremes in extremes_clusters:
				for element in extremes:
						__a           =			element
		__a           =			duplicate_indices - set(extreme_dict.keys()			)
		__a           =			dataset.filter(lambda lowerCAmelCase__    ,					lowerCAmelCase__			: idx not in remove_indices    ,					with_indices=lowerCAmelCase__			)

		# update duplicate_clusters
		for cluster in duplicate_clusters:
				for element in cluster:
						__a           =			element['''base_index'''] in extreme_dict
						if element["is_extreme"]:
								__a           =			extreme_dict[element['''base_index''']]['''copies''']

		print(f'''Original dataset size: {len(lowerCAmelCase__			)}'''			)
		print(f'''Number of duplicate clusters: {len(lowerCAmelCase__			)}'''			)
		print(f'''Files in duplicate cluster: {len(lowerCAmelCase__			)}'''			)
		print(f'''Unique files in duplicate cluster: {len(lowerCAmelCase__			)}'''			)
		print(f'''Filtered dataset size: {len(lowerCAmelCase__			)}'''			)

		return ds_filter, duplicate_clusters






