"""simple docstring"""




import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


__lowerCamelCase							 =     logging.getLogger(__name__)

__lowerCamelCase							 =     tf.data.AUTOTUNE






def      UpperCAmelCase    (		):






							"""simple docstring"""





							A__		   =							argparse.ArgumentParser(description='Train a masked language model on TPU.' )
							parser.add_argument(
							    '--pretrained_model_config'				,			type=_lowerCamelCase				,			default='roberta-base'				,			help='The model config to use. Note that we don\'t copy the model\'s weights, only the config!'				,			)
							parser.add_argument(
							    '--tokenizer'				,			type=_lowerCamelCase				,			default='unigram-tokenizer-wikitext'				,			help='The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'				,			)

							parser.add_argument(
							    '--per_replica_batch_size'				,			type=_lowerCamelCase				,			default=8				,			help='Batch size per TPU core.'				,			)

							parser.add_argument(
							    '--no_tpu'				,			action='store_true'				,			help='If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'				,			)

							parser.add_argument(
							    '--tpu_name'				,			type=_lowerCamelCase				,			help='Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'				,			default='local'				,			)

							parser.add_argument(
							    '--tpu_zone'				,			type=_lowerCamelCase				,			help='Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'				,			)

							parser.add_argument(
							    '--gcp_project'				,			type=_lowerCamelCase				,			help='Google cloud project name. Only used for non-Colab TPU nodes.' )

							parser.add_argument(
							    '--bfloat16'				,			action='store_true'				,			help='Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'				,			)

							parser.add_argument(
							    '--train_dataset'				,			type=_lowerCamelCase				,			help='Path to training dataset to load. If the path begins with `gs://`'
							    ' then the dataset will be loaded from a Google Cloud Storage bucket.'				,			)

							parser.add_argument(
							    '--shuffle_buffer_size'				,			type=_lowerCamelCase				,			default=2**18				,			help='Size of the shuffle buffer (in samples)'				,			)

							parser.add_argument(
							    '--eval_dataset'				,			type=_lowerCamelCase				,			help='Path to evaluation dataset to load. If the path begins with `gs://`'
							    ' then the dataset will be loaded from a Google Cloud Storage bucket.'				,			)

							parser.add_argument(
							    '--num_epochs'				,			type=_lowerCamelCase				,			default=1				,			help='Number of epochs to train for.'				,			)

							parser.add_argument(
							    '--learning_rate'				,			type=_lowerCamelCase				,			default=1E-4				,			help='Learning rate to use for training.'				,			)

							parser.add_argument(
							    '--weight_decay_rate'				,			type=_lowerCamelCase				,			default=1E-3				,			help='Weight decay rate to use for training.'				,			)

							parser.add_argument(
							    '--max_length'				,			type=_lowerCamelCase				,			default=512				,			help='Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'				,			)

							parser.add_argument(
							    '--mlm_probability'				,			type=_lowerCamelCase				,			default=0.1_5				,			help='Fraction of tokens to mask during training.'				,			)

							parser.add_argument('--output_dir'				,			type=_lowerCamelCase				,			required=_lowerCamelCase				,			help='Path to save model checkpoints to.' )
							parser.add_argument('--hub_model_id'				,			type=_lowerCamelCase				,			help='Model ID to upload to on the Hugging Face Hub.' )

							A__		   =							parser.parse_args()
							return args






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							try:
														if args.tpu_name:
																					A__		   =							tf.distribute.cluster_resolver.TPUClusterResolver(
																					    args.tpu_name				,			zone=args.tpu_zone				,			project=args.gcp_project )
														else:
																					A__		   =							tf.distribute.cluster_resolver.TPUClusterResolver()
							except ValueError:
														raise RuntimeError(
														    'Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '
														    '--gcp_project. When running on a TPU VM, use --tpu_name local.' )

							tf.config.experimental_connect_to_cluster(_lowerCamelCase )
							tf.tpu.experimental.initialize_tpu_system(_lowerCamelCase )

							return tpu






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							A__		   =							0
							for file in file_list:
														A__		   =							file.split('/' )[-1]
														A__		   =							re.search(r'-\d+-(\d+)\.tfrecord'				,			_lowerCamelCase ).group(1 )
														A__		   =							int(_lowerCamelCase )
														num_samples += sample_count

							return num_samples






def      UpperCAmelCase    (		UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__				,			UpperCamelCase__=None ):






							"""simple docstring"""





							A__		   =							count_samples(_lowerCamelCase )
							A__		   =							tf.data.Dataset.from_tensor_slices(_lowerCamelCase )
							if shuffle:
														A__		   =							dataset.shuffle(len(_lowerCamelCase ) )
							A__		   =							tf.data.TFRecordDataset(_lowerCamelCase				,			num_parallel_reads=_lowerCamelCase )
							# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
							A__		   =							dataset.apply(tf.data.experimental.assert_cardinality(_lowerCamelCase ) )
							A__		   =							dataset.map(_lowerCamelCase				,			num_parallel_calls=_lowerCamelCase )
							if shuffle:
														assert shuffle_buffer_size is not None
														A__		   =							dataset.shuffle(args.shuffle_buffer_size )
							A__		   =							dataset.batch(_lowerCamelCase				,			drop_remainder=_lowerCamelCase )
							A__		   =							dataset.map(_lowerCamelCase				,			num_parallel_calls=_lowerCamelCase )
							A__		   =							dataset.prefetch(_lowerCamelCase )
							return dataset






def      UpperCAmelCase    (		UpperCamelCase__ ):






							"""simple docstring"""





							if not args.no_tpu:
														A__		   =							initialize_tpu(_lowerCamelCase )
														A__		   =							tf.distribute.TPUStrategy(_lowerCamelCase )
							else:
														A__		   =							tf.distribute.OneDeviceStrategy(device='/gpu:0' )

							if args.bfloataa:
														tf.keras.mixed_precision.set_global_policy('mixed_bfloat16' )

							A__		   =							AutoTokenizer.from_pretrained(args.tokenizer )
							A__		   =							AutoConfig.from_pretrained(args.pretrained_model_config )
							A__		   =							tokenizer.vocab_size

							A__		   =							tf.io.gfile.glob(os.path.join(args.train_dataset				,			'*.tfrecord' ) )
							if not training_records:
														raise ValueError(F'''No .tfrecord files found in {args.train_dataset}.''' )
							A__		   =							tf.io.gfile.glob(os.path.join(args.eval_dataset				,			'*.tfrecord' ) )
							if not eval_records:
														raise ValueError(F'''No .tfrecord files found in {args.eval_dataset}.''' )

							A__		   =							count_samples(_lowerCamelCase )

							A__		   =							num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
							A__		   =							steps_per_epoch * args.num_epochs

							with strategy.scope():
														A__		   =							TFAutoModelForMaskedLM.from_config(_lowerCamelCase )
														model(model.dummy_inputs )  # Pass some dummy inputs through the model to ensure all the weights are built
														A__		   =							create_optimizer(
														    num_train_steps=_lowerCamelCase				,			num_warmup_steps=total_train_steps // 20				,			init_lr=args.learning_rate				,			weight_decay_rate=args.weight_decay_rate				,			)

														# Transformers models compute the right loss for their task by default when labels are passed, and will
														# use this for training unless you specify your own loss function in compile().
														model.compile(optimizer=_lowerCamelCase				,			metrics=['accuracy'] )

							def decode_fn(UpperCamelCase__ ):
														A__		   =							{
														    "input_ids": tf.io.FixedLenFeature(dtype=tf.intaa				,			shape=(args.max_length,) ),
														    "attention_mask": tf.io.FixedLenFeature(dtype=tf.intaa				,			shape=(args.max_length,) ),
														}
														return tf.io.parse_single_example(_lowerCamelCase				,			_lowerCamelCase )

							# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
							# use their methods in our data pipeline.
							A__		   =							DataCollatorForLanguageModeling(
							    tokenizer=_lowerCamelCase				,			mlm_probability=args.mlm_probability				,			mlm=_lowerCamelCase				,			return_tensors='tf' )

							def mask_with_collator(UpperCamelCase__ ):
														# TF really needs an isin() function
														A__		   =							(
														    ~tf.cast(batch['attention_mask']				,			tf.bool )
														    | (batch["input_ids"] == tokenizer.cls_token_id)
														    | (batch["input_ids"] == tokenizer.sep_token_id)
														)
														A__		   =							data_collator.tf_mask_tokens(
														    batch['input_ids']				,			vocab_size=len(_lowerCamelCase )				,			mask_token_id=tokenizer.mask_token_id				,			special_tokens_mask=_lowerCamelCase				,			)
														return batch

							A__		   =							args.per_replica_batch_size * strategy.num_replicas_in_sync

							A__		   =							prepare_dataset(
							    _lowerCamelCase				,			decode_fn=_lowerCamelCase				,			mask_fn=_lowerCamelCase				,			batch_size=_lowerCamelCase				,			shuffle=_lowerCamelCase				,			shuffle_buffer_size=args.shuffle_buffer_size				,			)

							A__		   =							prepare_dataset(
							    _lowerCamelCase				,			decode_fn=_lowerCamelCase				,			mask_fn=_lowerCamelCase				,			batch_size=_lowerCamelCase				,			shuffle=_lowerCamelCase				,			)

							A__		   =							[]
							if args.hub_model_id:
														callbacks.append(
														    PushToHubCallback(output_dir=args.output_dir				,			hub_model_id=args.hub_model_id				,			tokenizer=_lowerCamelCase ) )

							model.fit(
							    _lowerCamelCase				,			validation_data=_lowerCamelCase				,			epochs=args.num_epochs				,			callbacks=_lowerCamelCase				,			)

							model.save_pretrained(args.output_dir )


if __name__ == "__main__":
				__lowerCamelCase							 =     parse_args()
				main(args)










import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case		=  get_tests_dir("fixtures/test_sentencepiece_bpe.model")



class   UpperCAmelCase_    (       a   ,					unittest.TestCase):
							lowerCamelCase__   			=		BartphoTokenizer
							lowerCamelCase__   			=		False
							lowerCamelCase__   			=		True







							def    snake_case__      (   self):

														'''simple docstring'''

														super().setUp()

														_lowerCAmelCase     :							str    =    ["▁This", "▁is", "▁a", "▁t", "est"]
														_lowerCAmelCase     :							List[str]    =    dict(zip(__a,			range(len(__a))))
														_lowerCAmelCase     :							Optional[Any]    =    {"unk_token": "<unk>"}

														_lowerCAmelCase     :							Optional[int]    =    os.path.join(self.tmpdirname,			VOCAB_FILES_NAMES["monolingual_vocab_file"])
														with open(self.monolingual_vocab_file,			"w",			encoding="utf-8") as fp:
																					for token in vocab_tokens:
																												fp.write(f"{token} {vocab_tokens[token]}\n")

														_lowerCAmelCase     :							Optional[Any]    =    BartphoTokenizer(__a,			self.monolingual_vocab_file,			**self.special_tokens_map)
														tokenizer.save_pretrained(self.tmpdirname)







							def    snake_case__      (   self,			**__a):

														'''simple docstring'''

														kwargs.update(self.special_tokens_map)
														return BartphoTokenizer.from_pretrained(self.tmpdirname,			**__a)







							def    snake_case__      (   self,			__a):

														'''simple docstring'''

														_lowerCAmelCase     :							Union[str, Any]    =    "This is a là test"
														_lowerCAmelCase     :							Optional[int]    =    "This is a<unk><unk> test"
														return input_text, output_text







							def    snake_case__      (   self):

														'''simple docstring'''

														_lowerCAmelCase     :							Optional[int]    =    BartphoTokenizer(__a,			self.monolingual_vocab_file,			**self.special_tokens_map)
														_lowerCAmelCase     :							List[Any]    =    "This is a là test"
														_lowerCAmelCase     :							str    =    "▁This ▁is ▁a ▁l à ▁t est".split()
														_lowerCAmelCase     :							str    =    tokenizer.tokenize(__a)
														self.assertListEqual(__a,			__a)

														_lowerCAmelCase     :							Tuple    =    tokens + [tokenizer.unk_token]
														_lowerCAmelCase     :							List[str]    =    [4, 5, 6, 3, 3, 7, 8, 3]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(__a),			__a)
