





"""simple docstring"""





import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
						    SpeechaTextaDecoder,
						    SpeechaTextaForCausalLM,
						)





@require_torch
class 							_lowerCamelCase   :


						def __init__(self ,				__a ,				__a=99 ,				__a=13 ,				__a=16 ,				__a=7 ,				__a=True ,				__a=True ,				__a=True ,				__a=False ,				__a=True ,				__a=2 ,				__a=32 ,				__a=4 ,				__a=4 ,				__a=30 ,				__a=0 ,				__a=1 ,				__a=2 ,				__a=None ,				)			-> Optional[int]:
								UpperCamelCase =						parent
								UpperCamelCase =						batch_size
								UpperCamelCase =						decoder_seq_length
								# For common tests
								UpperCamelCase =						self.decoder_seq_length
								UpperCamelCase =						is_training
								UpperCamelCase =						use_attention_mask
								UpperCamelCase =						use_labels

								UpperCamelCase =						vocab_size
								UpperCamelCase =						d_model
								UpperCamelCase =						d_model
								UpperCamelCase =						decoder_layers
								UpperCamelCase =						decoder_layers
								UpperCamelCase =						decoder_ffn_dim
								UpperCamelCase =						decoder_attention_heads
								UpperCamelCase =						decoder_attention_heads
								UpperCamelCase =						eos_token_id
								UpperCamelCase =						bos_token_id
								UpperCamelCase =						pad_token_id
								UpperCamelCase =						decoder_start_token_id
								UpperCamelCase =						use_cache
								UpperCamelCase =						max_position_embeddings

								UpperCamelCase =						None
								UpperCamelCase =						decoder_seq_length
								UpperCamelCase =						2
								UpperCamelCase =						1


						def      snake_case_ (self		)			-> Optional[int]:
								UpperCamelCase =						ids_tensor([self.batch_size, self.decoder_seq_length] ,				self.vocab_size		)

								UpperCamelCase =						None
								if self.use_attention_mask:
										UpperCamelCase =						ids_tensor([self.batch_size, self.decoder_seq_length] ,				vocab_size=2		)

								UpperCamelCase =						None
								if self.use_labels:
										UpperCamelCase =						ids_tensor([self.batch_size, self.decoder_seq_length] ,				self.vocab_size		)

								UpperCamelCase =						SpeechaTextaConfig(
								    vocab_size=self.vocab_size ,				d_model=self.d_model ,				decoder_layers=self.decoder_layers ,				decoder_ffn_dim=self.decoder_ffn_dim ,				decoder_attention_heads=self.decoder_attention_heads ,				eos_token_id=self.eos_token_id ,				bos_token_id=self.bos_token_id ,				use_cache=self.use_cache ,				pad_token_id=self.pad_token_id ,				decoder_start_token_id=self.decoder_start_token_id ,				max_position_embeddings=self.max_position_embeddings ,				)

								return (
								    config,
								    input_ids,
								    attention_mask,
								    lm_labels,
								)


						def      snake_case_ (self ,				__a ,				__a ,				__a ,				__a ,				)			-> int:
								UpperCamelCase =						True
								UpperCamelCase =						SpeechaTextaDecoder(config=__a		).to(__a		).eval()
								UpperCamelCase =						input_ids[:2]

								input_ids[input_ids == 0] += 1
								# first forward pass
								UpperCamelCase =						model(__a ,				use_cache=__a		)
								UpperCamelCase =						model(__a		)
								UpperCamelCase =						model(__a ,				use_cache=__a		)

								self.parent.assertTrue(len(__a		) == len(__a		)		)
								self.parent.assertTrue(len(__a		) == len(__a		) + 1		)

								UpperCamelCase =						outputs["past_key_values"]

								# create hypothetical next token and extent to next_input_ids
								UpperCamelCase =						ids_tensor((2, 1) ,				config.vocab_size - 1		) + 1

								# append to next input_ids and
								UpperCamelCase =						torch.cat([input_ids, next_tokens] ,				dim=-1		)

								print(__a		)

								UpperCamelCase =						model(__a		)["last_hidden_state"]
								UpperCamelCase =						model(__a ,				past_key_values=__a		)["last_hidden_state"]

								# select random slice
								UpperCamelCase =						ids_tensor((1,) ,				output_from_past.shape[-1]		).item()
								UpperCamelCase =						output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
								UpperCamelCase =						output_from_past[:, 0, random_slice_idx].detach()

								# test that outputs are equal for slice
								assert torch.allclose(__a ,				__a ,				atol=1e-3		)




						def      snake_case_ (self		)			-> Tuple:
								UpperCamelCase =						self.prepare_config_and_inputs()
								(
								    (
								    UpperCamelCase
								) ,					(
								    UpperCamelCase
								) ,					(
								    UpperCamelCase
								) ,					(
								    UpperCamelCase
								) ,					
								) =						config_and_inputs

								UpperCamelCase =						{
								    "input_ids": input_ids,
								    "attention_mask": attention_mask,
								}
								return config, inputs_dict





@require_torch
class 							_lowerCamelCase   (							_lowercase	,      _lowercase	,      _lowercase	,      unittest.TestCase			):
						UpperCAmelCase_			=      (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
						UpperCAmelCase_			=      (SpeechaTextaForCausalLM,) if is_torch_available() else ()
						UpperCAmelCase_			=      {"text-generation": SpeechaTextaForCausalLM} if is_torch_available() else {}
						UpperCAmelCase_			=      True
						UpperCAmelCase_			=      False


						def      snake_case_ (self ,				)			-> str:
								UpperCamelCase =						SpeechaTextaStandaloneDecoderModelTester(self ,				is_training=__a		)
								UpperCamelCase =						ConfigTester(self ,				config_class=__a		)


						def      snake_case_ (self		)			-> Dict:
								pass


						def      snake_case_ (self		)			-> List[str]:
								pass


						def      snake_case_ (self		)			-> Any:
								pass


						def      snake_case_ (self		)			-> Any:
								self.config_tester.run_common_tests()


						def      snake_case_ (self		)			-> Optional[Any]:
								UpperCamelCase =						self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past(*__a		)




						def      snake_case_ (self		)			-> Union[str, Any]:
								return













"""simple docstring"""





import argparse
import logging

import datasets
from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer

from transformers import AlbertTokenizerFast


lowerCAmelCase__        =      logging.getLogger(__name__)


def    a__       (       ):




		"""simple docstring"""


		UpperCamelCase =						argparse.ArgumentParser(description="Train a unigram tokenizer on the wikitext dataset."     )
		parser.add_argument(
		    "--dataset_name"	,						type=_SCREAMING_SNAKE_CASE	,						default="wikitext"	,						help="Name of the training. Explore datasets at: hf.co/datasets."	,						)
		parser.add_argument(
		    "--dataset_config"	,						type=_SCREAMING_SNAKE_CASE	,						default="wikitext-103-raw-v1"	,						help="Configuration name of the dataset."     )
		parser.add_argument(
		    "--batch_size"	,						type=_SCREAMING_SNAKE_CASE	,						default=1_000	,						help="Batch size during training."	,						)
		parser.add_argument(
		    "--vocab_size"	,						type=_SCREAMING_SNAKE_CASE	,						default=10_048	,						help="Size of the desired vocabulary."	,						)
		parser.add_argument(
		    "--limit"	,						default=_SCREAMING_SNAKE_CASE	,						type=_SCREAMING_SNAKE_CASE	,						help="Limit the number of shards (used for debugging)."	,						)
		parser.add_argument(
		    "--export_to_hub"	,						action="store_true"	,						)

		UpperCamelCase =						parser.parse_args()
		return args


def    a__       (       _SCREAMING_SNAKE_CASE     ):




		"""simple docstring"""


		UpperCamelCase =						datasets.load_dataset(args.dataset_name	,						args.dataset_config	,						split="train"     )

		if args.limit is not None:
				UpperCamelCase =						min(len(_SCREAMING_SNAKE_CASE     )	,						args.limit     )
				UpperCamelCase =						dataset.select(range(_SCREAMING_SNAKE_CASE     )     )
				logger.info(F"Limiting the dataset to {args.limit} entries."     )

		def batch_iterator():
				for i in range(0	,						len(_SCREAMING_SNAKE_CASE     )	,						args.batch_size     ):
						yield dataset[i : i + args.batch_size]["text"]

    # Prepare the tokenizer.
		UpperCamelCase =						Tokenizer(Unigram()     )
		UpperCamelCase =						normalizers.Sequence([normalizers.Replace("``"	,						"\""     ), normalizers.Replace("''"	,						"\""     )]     )
		UpperCamelCase =						pre_tokenizers.Metaspace()

		# Prepare the trainer.
		UpperCamelCase =						UnigramTrainer(
		    unk_token="<unk>"	,						special_tokens=["[CLS]", "[SEP]", "<unk>", "<pad>", "[MASK]"]	,						vocab_size=args.vocab_size	,						)

		logger.info("Training the tokenizer."     )
		tokenizer.train_from_iterator(batch_iterator()	,						trainer=_SCREAMING_SNAKE_CASE     )
		logger.info("Tokenizer training complete!"     )

		UpperCamelCase =						tokenizer.token_to_id("[CLS]"     )
		UpperCamelCase =						tokenizer.token_to_id("[SEP]"     )
		UpperCamelCase =						processors.TemplateProcessing(
		    single="[CLS]:0 $A:0 [SEP]:0"	,						pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1"	,						special_tokens=[
		        ("[CLS]", cls_token_id),
		        ("[SEP]", sep_token_id),
		    ]	,						)
		UpperCamelCase =						decoders.Metaspace()

		if args.export_to_hub:
				logger.info("Exporting the trained tokenzier to Hub."     )
				UpperCamelCase =						AlbertTokenizerFast(tokenizer_object=_SCREAMING_SNAKE_CASE     )
				new_tokenizer.push_to_hub("unigram-tokenizer-dataset"     )


if __name__ == "__main__":
						lowerCAmelCase__        =      parse_args()
						main(args)






