


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig
def 	_lowerCamelCase  (		A_  :	Optional[int]							,  A_  :	List[Any]							,  A_  :	Optional[int]=0      )			->			Dict:

   '''simple docstring'''






   if name is None:
      UpperCamelCase__ :			Union[str, Any] 						=None
   else:
      UpperCamelCase__ :			Tuple 						="." * max(0							,  spaces - 2      ) + "# {:" + str(5_0 - spaces      ) + "s}"
      UpperCamelCase__ :			Tuple 						=fmt.format(A_      )

   # Print and recurse (if needed).
   if isinstance(A_							,  A_      ):
      if msg is not None:
         print(A_      )
      for k in val.keys():
         recursive_print(A_							,  val[k]							,  spaces + 2      )
   elif isinstance(A_							,  torch.Tensor      ):
      print(A_							,  ":"							,  val.size()      )
   else:
      print(A_							,  ":"							,  A_      )
def 	_lowerCamelCase  (		A_  :	Union[str, Any]							,  A_  :	Tuple							,  A_  :	Union[str, Any]							,  A_  :	str							,  A_  :	Union[str, Any]      )			->			int:

   '''simple docstring'''






   UpperCamelCase__ :			Optional[int] 						=param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      UpperCamelCase__ :			Optional[Any] 						=(num_heads, hidden_size, num_splits) + input_shape[1:]
      UpperCamelCase__ :			Dict 						=param.view(*A_      )
      UpperCamelCase__ :			str 						=param.transpose(0							,  2      )
      UpperCamelCase__ :			Optional[int] 						=param.transpose(1							,  2      ).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      UpperCamelCase__ :			str 						=(num_heads, num_splits, hidden_size) + input_shape[1:]
      UpperCamelCase__ :			Optional[int] 						=param.view(*A_      )
      UpperCamelCase__ :			Dict 						=param.transpose(0							,  1      ).contiguous()
   UpperCamelCase__ :			Union[str, Any] 						=param.view(*A_      )
   return param
def 	_lowerCamelCase  (		A_  :	Tuple							,  A_  :	Dict							,  A_  :	Union[str, Any]      )			->			Dict:

   '''simple docstring'''






   UpperCamelCase__ :			Tuple 						={}

   # old versions did not store training args
   UpperCamelCase__ :			Tuple 						=input_state_dict.get("args"							,  A_      )
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      UpperCamelCase__ :			Optional[int] 						=ds_args.tokenizer_type
      UpperCamelCase__ :			List[str] 						=ds_args.padded_vocab_size
      UpperCamelCase__ :			Optional[Any] 						=ds_args.max_position_embeddings
      UpperCamelCase__ :			Dict 						=ds_args.hidden_size
      UpperCamelCase__ :			Any 						=ds_args.num_layers
      UpperCamelCase__ :			str 						=ds_args.num_attention_heads
      UpperCamelCase__ :			Optional[int] 						=ds_args.ffn_hidden_size if "ffn_hidden_size" in ds_args else 4 * ds_args.hidden_size
      # pprint(config)

   # The number of heads.
   UpperCamelCase__ :			List[Any] 						=config.num_attention_heads
   # The hidden_size per head.
   UpperCamelCase__ :			Union[str, Any] 						=config.hidden_size // heads
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      UpperCamelCase__ :			List[str] 						=input_state_dict["checkpoint_version"]
   else:
      UpperCamelCase__ :			Optional[int] 						=0.0

   # The model.
   UpperCamelCase__ :			Any 						=input_state_dict["model"]
   # The language model.
   UpperCamelCase__ :			Union[str, Any] 						=model["language_model"]
   # The embeddings.
   UpperCamelCase__ :			List[Any] 						=lm["embedding"]

   # The word embeddings.
   UpperCamelCase__ :			List[Any] 						=embeddings["word_embeddings"]["weight"]
   # Truncate the embedding table to vocab_size rows.
   UpperCamelCase__ :			Optional[int] 						=word_embeddings[: config.vocab_size, :]
   # Store the word embeddings.
   UpperCamelCase__ :			Tuple 						=word_embeddings

   # The position embeddings.
   UpperCamelCase__ :			int 						=embeddings["position_embeddings"]["weight"]
   assert pos_embeddings.size(0      ) == config.max_position_embeddings and pos_embeddings.size(1      ) == config.hidden_size
   # Store the position embeddings.
   UpperCamelCase__ :			str 						=pos_embeddings

   # The token-type embeddings.
   UpperCamelCase__ :			Tuple 						=embeddings["tokentype_embeddings"]["weight"]
   # Store the position embeddings.
   UpperCamelCase__ :			Dict 						=tokentype_embeddings

   # The transformer.
   UpperCamelCase__ :			int 						=lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

   # The regex to extract layer names.
   UpperCamelCase__ :			List[str] 						=re.compile(R"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"      )

   # The simple map of names for "automated" rules.
   UpperCamelCase__ :			Optional[int] 						={
       "attention.dense": ".attention.output.dense.",
       "self_attention.dense": ".attention.output.dense.",
       "mlp.dense_h_to_4h": ".intermediate.dense.",
       "mlp.dense_4h_to_h": ".output.dense.",
   }

   # Keep track of the attention/query/value tensor.
   UpperCamelCase__ :			List[Any] 						=None

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      UpperCamelCase__ :			Union[str, Any] 						=layer_re.match(A_      )

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      UpperCamelCase__ :			Optional[int] 						=int(m.group(1      )      )
      # The name of the operation.
      UpperCamelCase__ :			List[str] 						=m.group(2      )
      # Is it a weight or a bias?
      UpperCamelCase__ :			Optional[int] 						=m.group(3      )

      # The name of the layer.
      UpperCamelCase__ :			Any 						=f'''bert.encoder.layer.{layer_idx}'''

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith("layernorm"      ):
         UpperCamelCase__ :			Tuple 						="attention.ln" if op_name.startswith("input"      ) else "ln"
         UpperCamelCase__ :			List[Any] 						=val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Make sure the QKV pointer is nil.
         assert attention_qkv_weight is None, ""

         UpperCamelCase__ :			List[str] 						=fix_query_key_value_ordering(A_							,  A_							,  3							,  A_							,  A_      )
         # Store the tensor as we need the bias as well to interleave QKV and biases.
         UpperCamelCase__ :			List[Any] 						=out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         # Make sure we read the weight tensor.
         assert attention_qkv_weight is not None, ""

         # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
         UpperCamelCase__ :			List[str] 						=attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
         UpperCamelCase__ :			int 						=attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
         UpperCamelCase__ :			Optional[Any] 						=attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

         UpperCamelCase__ :			List[Any] 						=fix_query_key_value_ordering(A_							,  A_							,  3							,  A_							,  A_      )
         # Split the bias.
         UpperCamelCase__ :			Union[str, Any] 						=out_val[0 * config.hidden_size : 1 * config.hidden_size]
         UpperCamelCase__ :			Dict 						=out_val[1 * config.hidden_size : 2 * config.hidden_size]
         UpperCamelCase__ :			Dict 						=out_val[2 * config.hidden_size : 3 * config.hidden_size]

         # Store.
         UpperCamelCase__ :			List[Any] 						=q
         UpperCamelCase__ :			Any 						=q_bias
         UpperCamelCase__ :			Union[str, Any] 						=k
         UpperCamelCase__ :			int 						=k_bias
         UpperCamelCase__ :			Any 						=v
         UpperCamelCase__ :			Tuple 						=v_bias

         # Clear the stored tensor.
         UpperCamelCase__ :			Optional[int] 						=None

      # Copy weights and biases as is.
      elif weight_or_bias in ["weight", "bias"]:
         UpperCamelCase__ :			Optional[int] 						=megatron_to_transformers[op_name]
         UpperCamelCase__ :			Union[str, Any] 						=val

    # The final layernorm.
   UpperCamelCase__ :			Optional[int] 						=transformer["final_layernorm.weight"]
   UpperCamelCase__ :			List[str] 						=transformer["final_layernorm.bias"]

   # The pooler.
   UpperCamelCase__ :			Dict 						=lm["pooler"]

   # Store the matrix and the bias.
   UpperCamelCase__ :			Any 						=pooler["dense.weight"]
   UpperCamelCase__ :			Optional[Any] 						=pooler["dense.bias"]

   # The LM head from Megatron (for RACE).
   UpperCamelCase__ :			Optional[Any] 						=model["lm_head"]

   # The transform matrix.
   UpperCamelCase__ :			Optional[int] 						=lm_head["dense.weight"]
   UpperCamelCase__ :			int 						=lm_head["dense.bias"]

   # The transform LN.
   UpperCamelCase__ :			List[Any] 						=lm_head["layernorm.weight"]
   UpperCamelCase__ :			Tuple 						=lm_head["layernorm.bias"]

   # For the decoder, we replicate the weights.
   UpperCamelCase__ :			List[str] 						=word_embeddings
   UpperCamelCase__ :			str 						=lm_head["bias"]

   # The classifier from Megatron (for MLNI).
   UpperCamelCase__ :			Tuple 						=model["binary_head"]

   # Store the classifier.
   UpperCamelCase__ :			Union[str, Any] 						=binary_head["weight"]
   UpperCamelCase__ :			Union[str, Any] 						=binary_head["bias"]

   # It should be done!
   return output_state_dict
def 	_lowerCamelCase  (		)			->			Optional[int]:

   '''simple docstring'''






   UpperCamelCase__ :			Union[str, Any] 						=argparse.ArgumentParser()
   parser.add_argument("--print-checkpoint-structure"							,  action="store_true"      )
   parser.add_argument("path_to_checkpoint"							,  type=A_							,  help="Path to the ZIP file containing the checkpoint"      )
   parser.add_argument(
       "--config_file"							,  default=""							,  type=A_							,  help="An optional config json file describing the pre-trained model."							,  )
   UpperCamelCase__ :			Any 						=parser.parse_args()

   # Extract the basename.
   UpperCamelCase__ :			Union[str, Any] 						=os.path.dirname(args.path_to_checkpoint      )

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(f'''Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'''      )
   if args.path_to_checkpoint.endswith(".zip"      ):
      with zipfile.ZipFile(args.path_to_checkpoint							,  "r"      ) as checkpoint:
         with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"      ) as pytorch_dict:
            UpperCamelCase__ :			Optional[Any] 						=torch.load(A_							,  map_location="cpu"      )
   else:
      UpperCamelCase__ :			Dict 						=torch.load(args.path_to_checkpoint							,  map_location="cpu"      )

   if args.config_file == "":
      # Default config of megatron-bert 345m
      UpperCamelCase__ :			Tuple 						=MegatronBertConfig()

      # different megatron-bert-*-345m models have different vocab sizes, so override the default
      # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
      UpperCamelCase__ :			List[Any] 						=input_state_dict["model"]["lm_head"]["bias"].numel()
   else:
      UpperCamelCase__ :			Tuple 						=MegatronBertConfig.from_json_file(args.config_file      )

   # Convert.
   print("Converting"      )
   UpperCamelCase__ :			List[str] 						=convert_megatron_checkpoint(A_							,  A_							,  A_      )

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(A_							,  A_      )

   # Store the config to file.
   print("Saving config"      )
   config.save_pretrained(A_      )

   # Store the state_dict to file.
   UpperCamelCase__ :			Any 						=os.path.join(A_							,  "pytorch_model.bin"      )
   print(f'''Saving checkpoint to "{output_checkpoint_file}"'''      )
   torch.save(A_							,  A_      )


####################################################################################################

if __name__ == "__main__":
     main()

####################################################################################################





import logging
import os
import sys
from dataclasses import dataclass, field
from typing import List, Optional

import torch
from datasets import Dataset
from torch import nn
from tqdm.auto import tqdm

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
    utils,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


__UpperCAmelCase      			=  """
Distills an NLI-based zero-shot classifier to a smaller, more efficient model with a fixed set of candidate class
names. Useful for speeding up zero-shot classification in cases where labeled training data is not available, but
when only a single fixed set of classes is needed. Takes a teacher NLI model, student classifier model, unlabeled
dataset, and set of K possible class names. Yields a single classifier with K outputs corresponding to the provided
class names.
"""

__UpperCAmelCase      			=  logging.getLogger(__name__)



@dataclass
class 							lowercase__:






      '''simple docstring'''







      snake_case__       =					field(
          default='''roberta-large-mnli'''			,							metadata={'''help''': '''The NLI/zero-shot teacher model to be distilled.'''}			)
      snake_case__       =					field(
          default='''This example is {}.'''			,							metadata={
              '''help''': (
                  '''Template used to turn class names into mock hypotheses for teacher NLI model. Must include {{}}'''
                  '''where class name is inserted.'''
              )
          }			,							)
      snake_case__       =					field(
          default=3_2			,							metadata={'''help''': '''Batch size for generating teacher predictions.'''}			)
      snake_case__       =					field(
          default=snake_case__			,							metadata={
              '''help''': (
                  '''Allow multiple classes to be true rather than forcing them to sum to 1 (sometimes called'''
                  '''multi-class multi-label classification).'''
              )
          }			,							)
      snake_case__       =					field(
          default=1.0			,							metadata={'''help''': '''Temperature applied to teacher softmax for distillation.'''}			)




@dataclass
class 							lowercase__:






      '''simple docstring'''







      snake_case__       =					field(
          default='''distilbert-base-uncased'''			,							metadata={'''help''': '''The NLI/zero-shot teacher model to be distilled.'''}			)




@dataclass
class 							lowercase__:






      '''simple docstring'''







      snake_case__       =					field(metadata={'''help''': '''Text file with one unlabeled instance per line.'''}			)
      snake_case__       =					field(metadata={'''help''': '''Text file with one class name per line.'''}			)
      snake_case__       =					field(
          default=snake_case__			,							metadata={'''help''': '''Whether to use one of the fast tokenizer (backed by the Rust tokenizers library) or not.'''}			,							)




@dataclass
class 							lowercase__(						snake_case__			):






      '''simple docstring'''







      snake_case__       =					field(
          default=snake_case__			,							metadata={'''help''': '''The output directory where the model predictions and checkpoints will be written.'''}			,							)
      snake_case__       =					field(
          default=3_2			,							metadata={'''help''': '''Batch size per GPU/TPU core/CPU for training.'''}			)
      snake_case__       =					field(
          default=1_2_8			,							metadata={'''help''': '''Batch size per GPU/TPU core/CPU for evaluation.'''}			)
      snake_case__       =					field(default=1.0			,							metadata={'''help''': '''Total number of training epochs to perform.'''}			)
      snake_case__       =					field(default=snake_case__			,							metadata={'''help''': '''Whether to run training of student model.'''}			)
      snake_case__       =					field(
          default=snake_case__			,							metadata={
              '''help''': (
                  '''Whether to evaluate the agreement of the final student predictions and the teacher predictions'''
                  '''after training.'''
              )
          }			,							)
      snake_case__       =					field(
          default=0			,							metadata={
              '''help''': (
                  '''Limit the total amount of checkpoints.'''
                  '''Deletes the older checkpoints in the output_dir. Default is 0 (no checkpoints).'''
              )
          }			,							)




class 							lowercase__(						snake_case__			):






      '''simple docstring'''







      def    UpperCAmelCase							(		self   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE   ,			__SCREAMING_SNAKE_CASE=False)					->						int:







         """simple docstring"""







         UpperCamelCase__ :			str 						=inputs["labels"]
         UpperCamelCase__ :			List[Any] 						=model(inputs["input_ids"]   ,			attention_mask=inputs["attention_mask"])
         UpperCamelCase__ :			List[str] 						=outputs[0]

         UpperCamelCase__ :			Optional[Any] 						=-torch.sum(target_p * logits.log_softmax(dim=-1)   ,			axis=-1).mean()

         if return_outputs:
            return loss, outputs

         return loss




def 	_lowerCamelCase  (		A_  :	List[Any]      )			->			List[Any]:

   '''simple docstring'''






   UpperCamelCase__ :			int 						=[]
   with open(A_							,  "r"      ) as f:
      for line in f:
         UpperCamelCase__ :			Any 						=line.strip()
         if len(A_      ) > 0:
            lines.append(A_      )
   return lines
def 	_lowerCamelCase  (		A_  :	Tuple							,  A_  :	Dict							,  A_  :	Tuple      )			->			Tuple:

   '''simple docstring'''






   UpperCamelCase__ :			Optional[int] 						=[]
   UpperCamelCase__ :			List[str] 						=[]
   for example in examples:
      for name in class_names:
         premises.append(A_      )
         hypotheses.append(hypothesis_template.format(A_      )      )
   return premises, hypotheses
def 	_lowerCamelCase  (		A_  :	int      )			->			Optional[Any]:

   '''simple docstring'''






   for label, ind in config.labelaid.items():
      if label.lower().startswith("entail"      ):
         return ind
   logger.warning("Could not identify entailment dimension from teacher config label2id. Setting to -1."      )
   return -1
def 	_lowerCamelCase  (		A_  :	str							,  A_  :	List[str]							,  A_  :	List[str]							,  A_  :	str							,  A_  :	int							,  A_  :	float							,  A_  :	bool							,  A_  :	bool							,  A_  :	bool							,  A_  :	bool							,  )			->			Tuple:

   '''simple docstring'''






   UpperCamelCase__ :			Any 						=AutoModelForSequenceClassification.from_pretrained(A_      )
   UpperCamelCase__ :			Any 						=model.config
   if not no_cuda and torch.cuda.is_available():
      UpperCamelCase__ :			Union[str, Any] 						=nn.DataParallel(model.cuda()      )
      batch_size *= len(model.device_ids      )
   UpperCamelCase__ :			Dict 						=AutoTokenizer.from_pretrained(A_							,  use_fast=A_      )

   UpperCamelCase__						,	UpperCamelCase__ :			Any 						=get_premise_hypothesis_pairs(A_							,  A_							,  A_      )
   UpperCamelCase__ :			Union[str, Any] 						=[]

   for i in tqdm(range(0							,  len(A_      )							,  A_      )      ):
      UpperCamelCase__ :			Any 						=premises[i : i + batch_size]
      UpperCamelCase__ :			Optional[int] 						=hypotheses[i : i + batch_size]

      UpperCamelCase__ :			Tuple 						=tokenizer(
          A_							,  A_							,  padding=A_							,  truncation="only_first"							,  return_tensors="pt"							,  )

      with torch.cuda.amp.autocast(enabled=A_      ):
         with torch.no_grad():
            UpperCamelCase__ :			Any 						=model(**A_      )
      logits.append(outputs.logits.detach().cpu().float()      )

   UpperCamelCase__ :			Any 						=get_entailment_id(A_      )
   UpperCamelCase__ :			Union[str, Any] 						=-1 if entail_id == 0 else 0
   UpperCamelCase__ :			Optional[int] 						=torch.cat(A_							,  dim=0      )  # N*K x 3
   UpperCamelCase__ :			List[Any] 						=logits.reshape(len(A_      )							,  len(A_      )							,  -1      )[..., [contr_id, entail_id]]  # N x K x 2

   if multi_label:
      # softmax over (contr, entail) logits for each class independently
      UpperCamelCase__ :			Optional[Any] 						=(nli_logits / temperature).softmax(-1      )
   else:
      # softmax over entail logits across classes s.t. class probabilities sum to 1.
      UpperCamelCase__ :			int 						=(nli_logits / temperature).softmax(1      )

   return nli_prob[..., 1]  # N x K
def 	_lowerCamelCase  (		)			->			Any:

   '''simple docstring'''






   UpperCamelCase__ :			Optional[Any] 						=HfArgumentParser(
       (DataTrainingArguments, TeacherModelArguments, StudentModelArguments, DistillTrainingArguments)							,  description=A_							,  )

   if len(sys.argv      ) == 2 and sys.argv[1].endswith(".json"      ):
      # If we pass only one argument to the script and it's the path to a json file,
      # let's parse it to get our arguments.
      UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			Union[str, Any] 						=parser.parse_json_file(
          json_file=os.path.abspath(sys.argv[1]      )      )
   else:
      UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__ :			Optional[Any] 						=parser.parse_args_into_dataclasses()

   # Detecting last checkpoint.
   UpperCamelCase__ :			str 						=None
   if os.path.isdir(training_args.output_dir      ) and training_args.do_train and not training_args.overwrite_output_dir:
      UpperCamelCase__ :			str 						=get_last_checkpoint(training_args.output_dir      )
      if last_checkpoint is None and len(os.listdir(training_args.output_dir      )      ) > 0:
         raise ValueError(
             f'''Output directory ({training_args.output_dir}) already exists and is not empty. '''
             "Use --overwrite_output_dir to overcome."      )
      elif last_checkpoint is not None:
         logger.info(
             f'''Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change '''
             "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."      )

    # Setup logging
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"							,  datefmt="%m/%d/%Y %H:%M:%S"							,  handlers=[logging.StreamHandler(sys.stdout      )]							,  )
   logger.setLevel(logging.INFO if is_main_process(training_args.local_rank      ) else logging.WARN      )

   # Log on each process the small summary:
   logger.warning(
       f'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}'''
       + f'''distributed training: {bool(training_args.local_rank != -1      )}, 16-bits training: {training_args.fpaa}'''      )
   # Set the verbosity to info of the Transformers logger (on main process only):
   if is_main_process(training_args.local_rank      ):
      utils.logging.set_verbosity_info()
      utils.logging.enable_default_handler()
      utils.logging.enable_explicit_format()

   if training_args.local_rank != -1:
      raise ValueError("Distributed training is not currently supported."      )
   if training_args.tpu_num_cores is not None:
      raise ValueError("TPU acceleration is not currently supported."      )

   logger.info(f'''Training/evaluation parameters {training_args}'''      )

   # Set seed before initializing model.
   set_seed(training_args.seed      )

   # 1. read in data
   UpperCamelCase__ :			Dict 						=read_lines(data_args.data_file      )
   UpperCamelCase__ :			Any 						=read_lines(data_args.class_names_file      )

   # 2. get teacher predictions and load into dataset
   logger.info("Generating predictions from zero-shot teacher model"      )
   UpperCamelCase__ :			Union[str, Any] 						=get_teacher_predictions(
       teacher_args.teacher_name_or_path							,  A_							,  A_							,  teacher_args.hypothesis_template							,  teacher_args.teacher_batch_size							,  teacher_args.temperature							,  teacher_args.multi_label							,  data_args.use_fast_tokenizer							,  training_args.no_cuda							,  training_args.fpaa							,  )
   UpperCamelCase__ :			Any 						=Dataset.from_dict(
       {
           "text": examples,
           "labels": teacher_soft_preds,
       }      )

   # 3. create student
   logger.info("Initializing student model"      )
   UpperCamelCase__ :			str 						=AutoModelForSequenceClassification.from_pretrained(
       student_args.student_name_or_path							,  num_labels=len(A_      )      )
   UpperCamelCase__ :			str 						=AutoTokenizer.from_pretrained(student_args.student_name_or_path							,  use_fast=data_args.use_fast_tokenizer      )
   UpperCamelCase__ :			List[Any] 						=dict(enumerate(A_      )      )
   UpperCamelCase__ :			str 						={label: i for i, label in enumerate(A_      )}

   # 4. train student on teacher predictions
   UpperCamelCase__ :			Any 						=dataset.map(A_							,  input_columns="text"      )
   dataset.set_format("torch"      )

   def compute_metrics(A_  :	Optional[Any]							,  A_  :	List[str]=False      ):
      UpperCamelCase__ :			Dict 						=p.predictions.argmax(-1      )
      UpperCamelCase__ :			Optional[Any] 						=p.label_ids.argmax(-1      )  # "label_ids" are actually distributions
      return {"agreement": (preds == proxy_labels).mean().item()}

   UpperCamelCase__ :			str 						=DistillationTrainer(
       model=A_							,  tokenizer=A_							,  args=A_							,  train_dataset=A_							,  compute_metrics=A_							,  )

   if training_args.do_train:
      logger.info("Training student model on teacher predictions"      )
      trainer.train()

   if training_args.do_eval:
      UpperCamelCase__ :			Any 						=trainer.evaluate(eval_dataset=A_      )["eval_agreement"]
      logger.info(f'''Agreement of student and teacher predictions: {agreement * 1_0_0:0.2f}%'''      )

   trainer.save_model()


if __name__ == "__main__":
     main()

