



import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


__UpperCAmelCase    	=  logging.getLogger(__name__)

require_version('''pytorch_lightning>=1.0.4''')

__UpperCAmelCase    	=  {
    '''base''': AutoModel,
    '''sequence-classification''': AutoModelForSequenceClassification,
    '''question-answering''': AutoModelForQuestionAnswering,
    '''pretraining''': AutoModelForPreTraining,
    '''token-classification''': AutoModelForTokenClassification,
    '''language-modeling''': AutoModelWithLMHead,
    '''summarization''': AutoModelForSeqaSeqLM,
    '''translation''': AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
__UpperCAmelCase    	=  {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
__UpperCAmelCase    	=  sorted(arg_to_scheduler.keys())
__UpperCAmelCase    	=  '''{''' + ''', '''.join(arg_to_scheduler_choices) + '''}'''


class 						lowerCamelCase__			(		pl.LightningModule ):


						def __init__(			self    :	List[Any]   ,							_a    :	argparse.Namespace   ,							_a    :	Optional[Any]=None   ,							_a    :	Tuple="base"   ,							_a    :	Any=None   ,							_a    :	str=None   ,							_a    :	Tuple=None   ,							**_a    :	Union[str, Any]   ,							):
													super().__init__()
													# TODO: move to self.save_hyperparameters()
													# self.save_hyperparameters()
													# can also expand arguments into trainer signature for easier reading

													self.save_hyperparameters(_a      )
													a__:							List[Any]									=0
													a__:							int									=Path(self.hparams.output_dir      )
													a__:							Tuple									=self.hparams.cache_dir if self.hparams.cache_dir else None
													if config is None:
																				a__:							Optional[int]									=AutoConfig.from_pretrained(
																				    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   ,							**({"num_labels": num_labels} if num_labels is not None else {})   ,							cache_dir=_a   ,							**_a   ,							)
													else:
																				a__:							PretrainedConfig									=config

													a__:							Any									=("encoder_layerdrop", "decoder_layerdrop", "dropout", "attention_dropout")
													for p in extra_model_params:
																				if getattr(self.hparams   ,							_a   ,							_a      ):
																											assert hasattr(self.config   ,							_a      ), F"model config doesn't have a `{p}` attribute"
																											setattr(self.config   ,							_a   ,							getattr(self.hparams   ,							_a      )      )

													if tokenizer is None:
																				a__:							str									=AutoTokenizer.from_pretrained(
																				    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   ,							cache_dir=_a   ,							)
													else:
																				a__:							PreTrainedTokenizer									=tokenizer
													a__:							int									=MODEL_MODES[mode]
													if model is None:
																				a__:							Optional[int]									=self.model_type.from_pretrained(
																				    self.hparams.model_name_or_path   ,							from_tf=bool(".ckpt" in self.hparams.model_name_or_path      )   ,							config=self.config   ,							cache_dir=_a   ,							)
													else:
																				a__:							Any									=model


						def 	_lowerCamelCase      (			self    :	Dict   ,							*_a    :	Any   ,							**_a    :	Optional[Any]      ):
													a__:							Tuple									=self.model_type.from_pretrained(*_a   ,							**_a      )


						def 	_lowerCamelCase      (			self    :	Any      ):
													a__:							Any									=arg_to_scheduler[self.hparams.lr_scheduler]
													a__:							Optional[Any]									=get_schedule_func(
													    self.opt   ,							num_warmup_steps=self.hparams.warmup_steps   ,							num_training_steps=self.total_steps()      )
													a__:							Optional[int]									={"scheduler": scheduler, "interval": "step", "frequency": 1}
													return scheduler


						def 	_lowerCamelCase      (			self    :	Optional[Any]      ):
													a__:							Dict									=self.model
													a__:							List[str]									=["bias", "LayerNorm.weight"]
													a__:							Dict									=[
													    {
													        "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay      )],
													        "weight_decay": self.hparams.weight_decay,
													    },
													    {
													        "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay      )],
													        "weight_decay": 0.0,
													    },
													]
													if self.hparams.adafactor:
																				a__:							int									=Adafactor(
																				    _a   ,							lr=self.hparams.learning_rate   ,							scale_parameter=_a   ,							relative_step=_a      )

													else:
																				a__:							Optional[int]									=AdamW(
																				    _a   ,							lr=self.hparams.learning_rate   ,							eps=self.hparams.adam_epsilon      )
													a__:							Optional[Any]									=optimizer

													a__:							Optional[int]									=self.get_lr_scheduler()

													return [optimizer], [scheduler]


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	Tuple   ,							_a    :	Optional[int]      ):
													return self.validation_step(_a   ,							_a      )


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	Dict      ):
													return self.validation_end(_a      )


						def 	_lowerCamelCase      (			self    :	int      ):
													a__:							Union[str, Any]									=max(1   ,							self.hparams.gpus      )  # TODO: consider num_tpu_cores
													a__:							List[str]									=self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
													return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs


						def 	_lowerCamelCase      (			self    :	Any   ,							_a    :	Optional[Any]      ):
													if mode == "test":
																				a__:							int									=len(self.test_dataloader().dataset      )
													else:
																				a__:							Dict									=self.get_dataloader("train"   ,							self.hparams.train_batch_size   ,							shuffle=_a      )
																				a__:							Any									=len(self.train_dataloader().dataset      )


						def 	_lowerCamelCase      (			self    :	List[str]   ,							_a    :	str   ,							_a    :	int   ,							_a    :	bool = False      ):
													raise NotImplementedError("You must implement this for your task"      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													return self.train_loader


						def 	_lowerCamelCase      (			self    :	str      ):
													return self.get_dataloader("dev"   ,							self.hparams.eval_batch_size   ,							shuffle=_a      )


						def 	_lowerCamelCase      (			self    :	Optional[int]      ):
													return self.get_dataloader("test"   ,							self.hparams.eval_batch_size   ,							shuffle=_a      )


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	int      ):
													return os.path.join(
													    self.hparams.data_dir   ,							"cached_{}_{}_{}".format(
													        _a   ,							list(filter(_a   ,							self.hparams.model_name_or_path.split("/"      )      )      ).pop()   ,							str(self.hparams.max_seq_length      )   ,							)   ,							)


						@pl.utilities.rank_zero_only
						def 	_lowerCamelCase      (			self    :	Dict   ,							_a    :	Dict[str, Any]      ):
													a__:							List[str]									=self.output_dir.joinpath("best_tfmr"      )
													a__:							Any									=self.step_count
													self.model.save_pretrained(_a      )
													self.tokenizer.save_pretrained(_a      )







						@staticmethod
						def 	_lowerCamelCase      (			_a    :	Optional[int]   ,							_a    :	Union[str, Any]      ):
													parser.add_argument(
													    "--model_name_or_path"   ,							default=_a   ,							type=_a   ,							required=_a   ,							help="Path to pretrained model or model identifier from huggingface.co/models"   ,							)
													parser.add_argument(
													    "--config_name"   ,							default=""   ,							type=_a   ,							help="Pretrained config name or path if not the same as model_name"      )
													parser.add_argument(
													    "--tokenizer_name"   ,							default=_a   ,							type=_a   ,							help="Pretrained tokenizer name or path if not the same as model_name"   ,							)
													parser.add_argument(
													    "--cache_dir"   ,							default=""   ,							type=_a   ,							help="Where do you want to store the pre-trained models downloaded from huggingface.co"   ,							)
													parser.add_argument(
													    "--encoder_layerdrop"   ,							type=_a   ,							help="Encoder layer dropout probability (Optional). Goes into model.config"   ,							)
													parser.add_argument(
													    "--decoder_layerdrop"   ,							type=_a   ,							help="Decoder layer dropout probability (Optional). Goes into model.config"   ,							)
													parser.add_argument(
													    "--dropout"   ,							type=_a   ,							help="Dropout probability (Optional). Goes into model.config"   ,							)
													parser.add_argument(
													    "--attention_dropout"   ,							type=_a   ,							help="Attention dropout probability (Optional). Goes into model.config"   ,							)
													parser.add_argument("--learning_rate"   ,							default=5e-5   ,							type=_a   ,							help="The initial learning rate for Adam."      )
													parser.add_argument(
													    "--lr_scheduler"   ,							default="linear"   ,							choices=_a   ,							metavar=_a   ,							type=_a   ,							help="Learning rate scheduler"   ,							)
													parser.add_argument("--weight_decay"   ,							default=0.0   ,							type=_a   ,							help="Weight decay if we apply some."      )
													parser.add_argument("--adam_epsilon"   ,							default=1e-8   ,							type=_a   ,							help="Epsilon for Adam optimizer."      )
													parser.add_argument("--warmup_steps"   ,							default=0   ,							type=_a   ,							help="Linear warmup over warmup_steps."      )
													parser.add_argument("--num_workers"   ,							default=4   ,							type=_a   ,							help="kwarg passed to DataLoader"      )
													parser.add_argument("--num_train_epochs"   ,							dest="max_epochs"   ,							default=3   ,							type=_a      )
													parser.add_argument("--train_batch_size"   ,							default=3_2   ,							type=_a      )
													parser.add_argument("--eval_batch_size"   ,							default=3_2   ,							type=_a      )
													parser.add_argument("--adafactor"   ,							action="store_true"      )


class 						lowerCamelCase__			(		pl.Callback ):


						def 	_lowerCamelCase      (			self    :	Any   ,							_a    :	Any   ,							_a    :	Optional[Any]      ):
													a__:							Optional[Any]									=trainer.lr_schedulers[0]["scheduler"]
													a__:							Tuple									={F"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()      )}
													pl_module.logger.log_metrics(_a      )


						def 	_lowerCamelCase      (			self    :	int   ,							_a    :	pl.Trainer   ,							_a    :	pl.LightningModule      ):
													rank_zero_info("***** Validation results *****"      )
													a__:							Tuple									=trainer.callback_metrics
													# Log results
													for key in sorted(_a      ):
																				if key not in ["log", "progress_bar"]:
																											rank_zero_info("{} = {}\n".format(_a   ,							str(metrics[key]      )      )      )







						def 	_lowerCamelCase      (			self    :	Union[str, Any]   ,							_a    :	pl.Trainer   ,							_a    :	pl.LightningModule      ):
													rank_zero_info("***** Test results *****"      )
													a__:							Any									=trainer.callback_metrics
													# Log and save results to file
													a__:							Any									=os.path.join(pl_module.hparams.output_dir   ,							"test_results.txt"      )
													with open(_a   ,							"w"      ) as writer:
																				for key in sorted(_a      ):
																											if key not in ["log", "progress_bar"]:
																																		rank_zero_info("{} = {}\n".format(_a   ,							str(metrics[key]      )      )      )
																																		writer.write("{} = {}\n".format(_a   ,							str(metrics[key]      )      )      )





def     __lowerCamelCase							(   __magic_name__						:	str  ,			__magic_name__						:	str    ):
							#  To allow all pl args uncomment the following line
							#  parser = pl.Trainer.add_argparse_args(parser)
							parser.add_argument(
							    "--output_dir"  ,			default=__magic_name__  ,			type=__magic_name__  ,			required=__magic_name__  ,			help="The output directory where the model predictions and checkpoints will be written."  ,			)
							parser.add_argument(
							    "--fp16"  ,			action="store_true"  ,			help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"  ,			)

							parser.add_argument(
							    "--fp16_opt_level"  ,			type=__magic_name__  ,			default="O2"  ,			help=(
							        "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
							        "See details at https://nvidia.github.io/apex/amp.html"
							    )  ,			)
							parser.add_argument("--n_tpu_cores"  ,			dest="tpu_cores"  ,			type=__magic_name__    )
							parser.add_argument("--max_grad_norm"  ,			dest="gradient_clip_val"  ,			default=1.0  ,			type=__magic_name__  ,			help="Max gradient norm"    )
							parser.add_argument("--do_train"  ,			action="store_true"  ,			help="Whether to run training."    )
							parser.add_argument("--do_predict"  ,			action="store_true"  ,			help="Whether to run predictions on the test set."    )
							parser.add_argument(
							    "--gradient_accumulation_steps"  ,			dest="accumulate_grad_batches"  ,			type=__magic_name__  ,			default=1  ,			help="Number of updates steps to accumulate before performing a backward/update pass."  ,			)
							parser.add_argument("--seed"  ,			type=__magic_name__  ,			default=42  ,			help="random seed for initialization"    )
							parser.add_argument(
							    "--data_dir"  ,			default=__magic_name__  ,			type=__magic_name__  ,			required=__magic_name__  ,			help="The input data dir. Should contain the training files for the CoNLL-2003 NER task."  ,			)







def     __lowerCamelCase							(   __magic_name__						:	BaseTransformer  ,			__magic_name__						:	argparse.Namespace  ,			__magic_name__						:	Dict=None  ,			__magic_name__						:	List[Any]=True  ,			__magic_name__						:	str=[]  ,			__magic_name__						:	int=None  ,			__magic_name__						:	str=None  ,			**__magic_name__						:	Optional[int]  ,			):
							pl.seed_everything(args.seed    )

							# init model
							a__:							str									=Path(model.hparams.output_dir    )
							odir.mkdir(exist_ok=__magic_name__    )

							# add custom checkpoints
							if checkpoint_callback is None:
														a__:							Any									=pl.callbacks.ModelCheckpoint(
														    filepath=args.output_dir  ,			prefix="checkpoint"  ,			monitor="val_loss"  ,			mode="min"  ,			save_top_k=1    )
							if early_stopping_callback:
														extra_callbacks.append(__magic_name__    )
							if logging_callback is None:
														a__:							Dict									=LoggingCallback()

							a__:							List[Any]									={}

							# TODO: remove with PyTorch 1.6 since pl uses native amp
							if args.fpaa:
														a__:							str									=16
														a__:							int									=args.fpaa_opt_level

							if args.gpus > 1:
														a__:							Union[str, Any]									="ddp"

							a__:							List[str]									=args.accumulate_grad_batches
							a__:							Optional[int]									=extra_train_kwargs.get("accelerator"  ,			__magic_name__    )
							a__:							Any									=extra_train_kwargs.get("profiler"  ,			__magic_name__    )

							a__:							Optional[int]									=pl.Trainer.from_argparse_args(
							    __magic_name__  ,			weights_summary=__magic_name__  ,			callbacks=[logging_callback] + extra_callbacks  ,			logger=__magic_name__  ,			checkpoint_callback=__magic_name__  ,			**__magic_name__  ,			)

							if args.do_train:
														trainer.fit(__magic_name__    )

							return trainer










import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
		from transformers import WhisperFeatureExtractor, WhisperProcessor


__UpperCAmelCase    	=  5_03_58
__UpperCAmelCase    	=  5_03_62


@require_torch
@require_torchaudio
@require_sentencepiece
class 						lowerCamelCase__			(		unittest.TestCase ):


						def 	_lowerCamelCase      (			self    :	List[Any]      ):
													a__:							str									="openai/whisper-small.en"
													a__:							Union[str, Any]									=tempfile.mkdtemp()


						def 	_lowerCamelCase      (			self    :	Optional[Any]   ,							**_a    :	Optional[int]      ):
													return WhisperTokenizer.from_pretrained(self.checkpoint   ,							**_a      )


						def 	_lowerCamelCase      (			self    :	str   ,							**_a    :	Tuple      ):
													return WhisperFeatureExtractor.from_pretrained(self.checkpoint   ,							**_a      )


						def 	_lowerCamelCase      (			self    :	Union[str, Any]      ):
													shutil.rmtree(self.tmpdirname      )


						def 	_lowerCamelCase      (			self    :	int      ):
													a__:							str									=self.get_tokenizer()
													a__:							Union[str, Any]									=self.get_feature_extractor()

													a__:							int									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )

													processor.save_pretrained(self.tmpdirname      )
													a__:							List[Any]									=WhisperProcessor.from_pretrained(self.tmpdirname      )

													self.assertEqual(processor.tokenizer.get_vocab()   ,							tokenizer.get_vocab()      )
													self.assertIsInstance(processor.tokenizer   ,							_a      )

													self.assertEqual(processor.feature_extractor.to_json_string()   ,							feature_extractor.to_json_string()      )
													self.assertIsInstance(processor.feature_extractor   ,							_a      )


						def 	_lowerCamelCase      (			self    :	Dict      ):
													a__:							str									=WhisperProcessor(tokenizer=self.get_tokenizer()   ,							feature_extractor=self.get_feature_extractor()      )
													processor.save_pretrained(self.tmpdirname      )

													a__:							Tuple									=self.get_tokenizer(bos_token="(BOS)"   ,							eos_token="(EOS)"      )
													a__:							Optional[int]									=self.get_feature_extractor(do_normalize=_a   ,							padding_value=1.0      )

													a__:							Any									=WhisperProcessor.from_pretrained(
													    self.tmpdirname   ,							bos_token="(BOS)"   ,							eos_token="(EOS)"   ,							do_normalize=_a   ,							padding_value=1.0      )

													self.assertEqual(processor.tokenizer.get_vocab()   ,							tokenizer_add_kwargs.get_vocab()      )
													self.assertIsInstance(processor.tokenizer   ,							_a      )

													self.assertEqual(processor.feature_extractor.to_json_string()   ,							feature_extractor_add_kwargs.to_json_string()      )
													self.assertIsInstance(processor.feature_extractor   ,							_a      )


						def 	_lowerCamelCase      (			self    :	Tuple      ):
													a__:							Optional[int]									=self.get_feature_extractor()
													a__:							str									=self.get_tokenizer()

													a__:							int									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )

													a__:							List[str]									=floats_list((3, 1_0_0_0)      )

													a__:							Optional[int]									=feature_extractor(_a   ,							return_tensors="np"      )
													a__:							Optional[int]									=processor(_a   ,							return_tensors="np"      )

													for key in input_feat_extract.keys():
																				self.assertAlmostEqual(input_feat_extract[key].sum()   ,							input_processor[key].sum()   ,							delta=1e-2      )


						def 	_lowerCamelCase      (			self    :	Optional[Any]      ):
													a__:							Dict									=self.get_feature_extractor()
													a__:							int									=self.get_tokenizer()

													a__:							List[Any]									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )

													a__:							List[Any]									="This is a test string"

													a__:							Tuple									=processor(text=_a      )

													a__:							int									=tokenizer(_a      )

													for key in encoded_tok.keys():
																				self.assertListEqual(encoded_tok[key]   ,							encoded_processor[key]      )


						def 	_lowerCamelCase      (			self    :	Any      ):
													a__:							int									=self.get_feature_extractor()
													a__:							List[str]									=self.get_tokenizer()

													a__:							List[str]									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )

													a__:							List[Any]									=[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

													a__:							List[Any]									=processor.batch_decode(_a      )
													a__:							int									=tokenizer.batch_decode(_a      )

													self.assertListEqual(_a   ,							_a      )


						def 	_lowerCamelCase      (			self    :	List[str]      ):
													a__:							List[Any]									=self.get_feature_extractor()
													a__:							Any									=self.get_tokenizer()

													a__:							Optional[int]									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )

													self.assertListEqual(
													    processor.model_input_names   ,							feature_extractor.model_input_names   ,							msg="`processor` and `feature_extractor` model input names do not match"   ,							)


						def 	_lowerCamelCase      (			self    :	List[Any]      ):
													a__:							Any									=self.get_feature_extractor()
													a__:							str									=self.get_tokenizer()

													a__:							Dict									=WhisperProcessor(tokenizer=_a   ,							feature_extractor=_a      )
													a__:							List[Any]									=processor.get_decoder_prompt_ids(task="transcribe"   ,							no_timestamps=_a      )

													self.assertIsInstance(_a   ,							_a      )
													for ids in forced_decoder_ids:
																				self.assertIsInstance(_a   ,							(list, tuple)      )

													a__:							Any									=[TRANSCRIBE, NOTIMESTAMPS]
													self.assertListEqual([ids[-1] for ids in forced_decoder_ids]   ,							_a      )


						def 	_lowerCamelCase      (			self    :	str      ):
													a__:							Tuple									=WhisperProcessor(tokenizer=self.get_tokenizer()   ,							feature_extractor=self.get_feature_extractor()      )
													a__:							List[Any]									=processor.get_prompt_ids("Mr. Quilter"      )
													a__:							Optional[Any]									=processor.tokenizer.decode(_a      )

													self.assertListEqual(prompt_ids.tolist()   ,							[5_0_3_6_0, 1_7_7_0, 1_3, 2_2_6_4, 3_4_6, 3_5_3]      )
													self.assertEqual(_a   ,							"<|startofprev|> Mr. Quilter"      )


						def 	_lowerCamelCase      (			self    :	List[str]      ):
													a__:							int									=WhisperProcessor(tokenizer=self.get_tokenizer()   ,							feature_extractor=self.get_feature_extractor()      )
													a__:							List[str]									=processor.get_prompt_ids(""      )
													a__:							Any									=processor.tokenizer.decode(_a      )

													self.assertListEqual(prompt_ids.tolist()   ,							[5_0_3_6_0, 2_2_0]      )
													self.assertEqual(_a   ,							"<|startofprev|> "      )







						def 	_lowerCamelCase      (			self    :	Any      ):
													a__:							Tuple									=WhisperProcessor(tokenizer=self.get_tokenizer()   ,							feature_extractor=self.get_feature_extractor()      )

													def _test_prompt_error_raised_helper(_a    :	Union[str, Any]   ,							_a    :	List[str]      ):
																				with pytest.raises(_a      ) as excinfo:
																											processor.get_prompt_ids(_a      )
																				a__:							List[Any]									=F"Encountered text in the prompt corresponding to disallowed special token: {special_token}."
																				self.assertEqual(_a   ,							str(excinfo.value      )      )

													_test_prompt_error_raised_helper("<|startofprev|> test"   ,							"<|startofprev|>"      )
													_test_prompt_error_raised_helper("test <|notimestamps|>"   ,							"<|notimestamps|>"      )
													_test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"   ,							"<|zh|>"      )





