




'''simple docstring'''

import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
		from transformers import WhisperFeatureExtractor, WhisperProcessor


a__         =     50_358
a__         =     50_362





@require_torch
@require_torchaudio
@require_sentencepiece
class __magic_name__(						unittest.TestCase      ):






						def       __lowerCAmelCase(   self						:						Optional[int]       ):



									'''simple docstring'''






									snake_case__          =						"""openai/whisper-small.en"""
									snake_case__          =						tempfile.mkdtemp()






						def       __lowerCAmelCase(   self						:						List[str]					,   **__UpperCamelCase						:						Dict       ):



									'''simple docstring'''






									return WhisperTokenizer.from_pretrained(self.checkpoint					,   **__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Dict					,   **__UpperCamelCase						:						Union[str, Any]       ):



									'''simple docstring'''






									return WhisperFeatureExtractor.from_pretrained(self.checkpoint					,   **__UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									shutil.rmtree(self.tmpdirname       )






						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									snake_case__          =						self.get_tokenizer()
									snake_case__          =						self.get_feature_extractor()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )

									processor.save_pretrained(self.tmpdirname       )
									snake_case__          =						WhisperProcessor.from_pretrained(self.tmpdirname       )

									self.assertEqual(processor.tokenizer.get_vocab()					,   tokenizer.get_vocab()       )
									self.assertIsInstance(processor.tokenizer					,   __UpperCamelCase       )

									self.assertEqual(processor.feature_extractor.to_json_string()					,   feature_extractor.to_json_string()       )
									self.assertIsInstance(processor.feature_extractor					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						List[Any]       ):



									'''simple docstring'''






									snake_case__          =						WhisperProcessor(tokenizer=self.get_tokenizer()					,   feature_extractor=self.get_feature_extractor()       )
									processor.save_pretrained(self.tmpdirname       )

									snake_case__          =						self.get_tokenizer(bos_token="""(BOS)"""					,   eos_token="""(EOS)"""       )
									snake_case__          =						self.get_feature_extractor(do_normalize=__UpperCamelCase					,   padding_value=1.0       )

									snake_case__          =						WhisperProcessor.from_pretrained(
									    self.tmpdirname					,   bos_token="""(BOS)"""					,   eos_token="""(EOS)"""					,   do_normalize=__UpperCamelCase					,   padding_value=1.0       )

									self.assertEqual(processor.tokenizer.get_vocab()					,   tokenizer_add_kwargs.get_vocab()       )
									self.assertIsInstance(processor.tokenizer					,   __UpperCamelCase       )

									self.assertEqual(processor.feature_extractor.to_json_string()					,   feature_extractor_add_kwargs.to_json_string()       )
									self.assertIsInstance(processor.feature_extractor					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						self.get_feature_extractor()
									snake_case__          =						self.get_tokenizer()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )

									snake_case__          =						floats_list((3, 1_0_0_0)       )

									snake_case__          =						feature_extractor(__UpperCamelCase					,   return_tensors="""np"""       )
									snake_case__          =						processor(__UpperCamelCase					,   return_tensors="""np"""       )

									for key in input_feat_extract.keys():
												self.assertAlmostEqual(input_feat_extract[key].sum()					,   input_processor[key].sum()					,   delta=1E-2       )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.get_feature_extractor()
									snake_case__          =						self.get_tokenizer()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )

									snake_case__          =						"""This is a test string"""

									snake_case__          =						processor(text=__UpperCamelCase       )

									snake_case__          =						tokenizer(__UpperCamelCase       )

									for key in encoded_tok.keys():
												self.assertListEqual(encoded_tok[key]					,   encoded_processor[key]       )






						def       __lowerCAmelCase(   self						:						str       ):



									'''simple docstring'''






									snake_case__          =						self.get_feature_extractor()
									snake_case__          =						self.get_tokenizer()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )

									snake_case__          =						[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									snake_case__          =						processor.batch_decode(__UpperCamelCase       )
									snake_case__          =						tokenizer.batch_decode(__UpperCamelCase       )

									self.assertListEqual(__UpperCamelCase					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						List[str]       ):



									'''simple docstring'''






									snake_case__          =						self.get_feature_extractor()
									snake_case__          =						self.get_tokenizer()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )

									self.assertListEqual(
									    processor.model_input_names					,   feature_extractor.model_input_names					,   msg="""`processor` and `feature_extractor` model input names do not match"""					,   )






						def       __lowerCAmelCase(   self						:						Union[str, Any]       ):



									'''simple docstring'''






									snake_case__          =						self.get_feature_extractor()
									snake_case__          =						self.get_tokenizer()

									snake_case__          =						WhisperProcessor(tokenizer=__UpperCamelCase					,   feature_extractor=__UpperCamelCase       )
									snake_case__          =						processor.get_decoder_prompt_ids(task="""transcribe"""					,   no_timestamps=__UpperCamelCase       )

									self.assertIsInstance(__UpperCamelCase					,   __UpperCamelCase       )
									for ids in forced_decoder_ids:
												self.assertIsInstance(__UpperCamelCase					,   (list, tuple)       )

									snake_case__          =						[TRANSCRIBE, NOTIMESTAMPS]
									self.assertListEqual([ids[-1] for ids in forced_decoder_ids]					,   __UpperCamelCase       )






						def       __lowerCAmelCase(   self						:						Any       ):



									'''simple docstring'''






									snake_case__          =						WhisperProcessor(tokenizer=self.get_tokenizer()					,   feature_extractor=self.get_feature_extractor()       )
									snake_case__          =						processor.get_prompt_ids("""Mr. Quilter"""       )
									snake_case__          =						processor.tokenizer.decode(__UpperCamelCase       )

									self.assertListEqual(prompt_ids.tolist()					,   [5_0_3_6_0, 1_7_7_0, 1_3, 2_2_6_4, 3_4_6, 3_5_3]       )
									self.assertEqual(__UpperCamelCase					,   """<|startofprev|> Mr. Quilter"""       )






						def       __lowerCAmelCase(   self						:						Optional[Any]       ):



									'''simple docstring'''






									snake_case__          =						WhisperProcessor(tokenizer=self.get_tokenizer()					,   feature_extractor=self.get_feature_extractor()       )
									snake_case__          =						processor.get_prompt_ids(""""""       )
									snake_case__          =						processor.tokenizer.decode(__UpperCamelCase       )

									self.assertListEqual(prompt_ids.tolist()					,   [5_0_3_6_0, 2_2_0]       )
									self.assertEqual(__UpperCamelCase					,   """<|startofprev|> """       )







						def       __lowerCAmelCase(   self						:						int       ):



									'''simple docstring'''






									snake_case__          =						WhisperProcessor(tokenizer=self.get_tokenizer()					,   feature_extractor=self.get_feature_extractor()       )

									def _test_prompt_error_raised_helper(__UpperCamelCase						:						Optional[int]					,   __UpperCamelCase						:						List[str]       ):
												with pytest.raises(__UpperCamelCase       ) as excinfo:
															processor.get_prompt_ids(__UpperCamelCase       )
												snake_case__          =						f"""Encountered text in the prompt corresponding to disallowed special token: {special_token}."""
												self.assertEqual(__UpperCamelCase					,   str(excinfo.value       )       )

									_test_prompt_error_raised_helper("""<|startofprev|> test"""					,   """<|startofprev|>"""       )
									_test_prompt_error_raised_helper("""test <|notimestamps|>"""					,   """<|notimestamps|>"""       )
									_test_prompt_error_raised_helper("""test <|zh|> test <|transcribe|>"""					,   """<|zh|>"""       )




"""simple docstring"""


import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
				print("""#################### AVAILABLE DEVICES ####################""")
				print(jax.devices())
				print("""###########################################################""")

				# setup for wandb sweep
				snake_case_								=			Args()
				snake_case_								=			wandb.init(project="""bigbird-natural-questions""", config=args.__dict__)
				snake_case_								=			dict(logger.config)
				del wandb_args["batch_size"]
				snake_case_								=			replace(args, **wandb_args)
				snake_case_								=			args.base_dir + """-""" + wandb.run.id
				snake_case_								=			replace(args, base_dir=base_dir)
				print(args)

				snake_case_								=			load_dataset("""json""", data_files=args.tr_data_path)["""train"""]
				snake_case_								=			load_dataset("""json""", data_files=args.val_data_path)["""train"""]

				# drop extra batch for now
				snake_case_								=			range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
				snake_case_								=			tr_dataset.shuffle().select(indices)
				snake_case_								=			range(len(val_dataset) - len(val_dataset) % args.batch_size)
				snake_case_								=			val_dataset.shuffle().select(indices)

				if os.environ.get("""TRAIN_ON_SMALL""", """false""") == "true":
								snake_case_								=			tr_dataset.shuffle().select(range(8_0000))
								snake_case_								=			val_dataset.shuffle().select(range(8000))

				print(tr_dataset)
				print(val_dataset)

				snake_case_								=			FlaxBigBirdForNaturalQuestions.from_pretrained(
				    args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
				)
				snake_case_								=			BigBirdTokenizerFast.from_pretrained(args.model_id)
				snake_case_								=			DataCollator(pad_id=tokenizer.pad_token_id, max_length=4096)

				snake_case_								=			{
				    """lr""": args.lr,
				    """init_lr""": args.init_lr,
				    """warmup_steps""": args.warmup_steps,
				    """num_train_steps""": args.max_epochs * (len(tr_dataset) // args.batch_size),
				    """weight_decay""": args.weight_decay,
				}
				snake_case_	,  snake_case_								=			build_tx(**tx_args)

				snake_case_								=			Trainer(
				    args=args,
				    data_collator=data_collator,
				    model_save_fn=model.save_pretrained,
				    train_step_fn=train_step,
				    val_step_fn=val_step,
				    logger=logger,
				    scheduler_fn=lr,
				)

				snake_case_								=			None
				snake_case_								=			trainer.create_state(model, tx, num_train_steps=tx_args["""num_train_steps"""], ckpt_dir=ckpt_dir)
				try:
								trainer.train(state, tr_dataset, val_dataset)
				except KeyboardInterrupt:
								print("""Oooops; TRAINING STOPPED UNFORTUNATELY""")

				print("""SAVING WEIGHTS IN `final-weights`""")
				snake_case_								=			jax_utils.unreplicate(state.params)
				model.save_pretrained(os.path.join(args.base_dir, """final-weights"""), params=params)





