




"""simple docstring"""


import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					lowercase(     __a       ,			unittest.TestCase ):







 '''simple docstring'''





 lowercase__ 					=					CodeGenTokenizer
 lowercase__ 					=					CodeGenTokenizerFast
 lowercase__ 					=					True
 lowercase__ 					=					{"add_prefix_space": True}
 lowercase__ 					=					False


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   _snake_case							:		Tuple								=  [
       """l""",
       """o""",
       """w""",
       """e""",
       """r""",
       """s""",
       """t""",
       """i""",
       """d""",
       """n""",
       """\u0120""",
       """\u0120l""",
       """\u0120n""",
       """\u0120lo""",
       """\u0120low""",
       """er""",
       """\u0120lowest""",
       """\u0120newer""",
       """\u0120wider""",
       """<unk>""",
       """<|endoftext|>""",
   ]
   _snake_case							:		Tuple								=  dict(zip(a_,      range(len(a_		)		)		)		)
   _snake_case							:		str								=  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
   _snake_case							:		List[Any]								=  {"""unk_token""": """<unk>"""}

   _snake_case							:		Optional[int]								=  os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES["""vocab_file"""]		)
   _snake_case							:		Optional[Any]								=  os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES["""merges_file"""]		)
   with open(self.vocab_file,      """w""",      encoding="""utf-8"""		) as fp:
     fp.write(json.dumps(a_		) + """\n"""		)
   with open(self.merges_file,      """w""",      encoding="""utf-8"""		) as fp:
     fp.write("""\n""".join(a_		)		)


 def  UpperCamelCase_   (				self:		Any,      **a_:		int		):




   '''simple docstring'''



   kwargs.update(self.special_tokens_map		)
   return CodeGenTokenizer.from_pretrained(self.tmpdirname,      **a_		)


 def  UpperCamelCase_   (				self:		Any,      **a_:		str		):




   '''simple docstring'''



   kwargs.update(self.special_tokens_map		)
   return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,      **a_		)


 def  UpperCamelCase_   (				self:		Union[str, Any],      a_:		Dict		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  """lower newer"""
   _snake_case							:		Tuple								=  """lower newer"""
   return input_text, output_text


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  CodeGenTokenizer(self.vocab_file,      self.merges_file,      **self.special_tokens_map		)
   _snake_case							:		Optional[Any]								=  """lower newer"""
   _snake_case							:		Optional[int]								=  ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
   _snake_case							:		int								=  tokenizer.tokenize(a_,      add_prefix_space=a_		)
   self.assertListEqual(a_,      a_		)

   _snake_case							:		str								=  tokens + [tokenizer.unk_token]
   _snake_case							:		Optional[int]								=  [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(a_		),      a_		)


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   if not self.test_rust_tokenizer:
     return

   _snake_case							:		int								=  self.get_tokenizer()
   _snake_case							:		int								=  self.get_rust_tokenizer(add_prefix_space=a_		)

   _snake_case							:		Dict								=  """lower newer"""

   # Testing tokenization
   _snake_case							:		Dict								=  tokenizer.tokenize(a_,      add_prefix_space=a_		)
   _snake_case							:		List[str]								=  rust_tokenizer.tokenize(a_		)
   self.assertListEqual(a_,      a_		)

   # Testing conversion to ids without special tokens
   _snake_case							:		Optional[Any]								=  tokenizer.encode(a_,      add_special_tokens=a_,      add_prefix_space=a_		)
   _snake_case							:		Tuple								=  rust_tokenizer.encode(a_,      add_special_tokens=a_		)
   self.assertListEqual(a_,      a_		)

   # Testing conversion to ids with special tokens
   _snake_case							:		Tuple								=  self.get_rust_tokenizer(add_prefix_space=a_		)
   _snake_case							:		int								=  tokenizer.encode(a_,      add_prefix_space=a_		)
   _snake_case							:		Optional[Any]								=  rust_tokenizer.encode(a_		)
   self.assertListEqual(a_,      a_		)

   # Testing the unknown token
   _snake_case							:		Tuple								=  tokens + [rust_tokenizer.unk_token]
   _snake_case							:		List[Any]								=  [14, 15, 10, 9, 3, 2, 15, 19]
   self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a_		),      a_		)


 def  UpperCamelCase_   (				self:		Dict,      *a_:		Dict,      **a_:		int		):




   '''simple docstring'''



   pass


 def  UpperCamelCase_   (				self:		int,      a_:		List[Any]=15		):




   '''simple docstring'''



   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
       _snake_case							:		List[Any]								=  self.rust_tokenizer_class.from_pretrained(a_,      **a_		)

       # Simple input
       _snake_case							:		Any								=  """This is a simple input"""
       _snake_case							:		Optional[int]								=  ["""This is a simple input 1""", """This is a simple input 2"""]
       _snake_case							:		Optional[int]								=  ("""This is a simple input""", """This is a pair""")
       _snake_case							:		Optional[Any]								=  [
           ("""This is a simple input 1""", """This is a simple input 2"""),
           ("""This is a simple pair 1""", """This is a simple pair 2"""),
       ]

       # Simple input tests
       self.assertRaises(a_,      tokenizer_r.encode,      a_,      max_length=a_,      padding="""max_length"""		)

       # Simple input
       self.assertRaises(a_,      tokenizer_r.encode_plus,      a_,      max_length=a_,      padding="""max_length"""		)

       # Simple input
       self.assertRaises(
           a_,      tokenizer_r.batch_encode_plus,      a_,      max_length=a_,      padding="""max_length""",      )

       # Pair input
       self.assertRaises(a_,      tokenizer_r.encode,      a_,      max_length=a_,      padding="""max_length"""		)

       # Pair input
       self.assertRaises(a_,      tokenizer_r.encode_plus,      a_,      max_length=a_,      padding="""max_length"""		)

       # Pair input
       self.assertRaises(
           a_,      tokenizer_r.batch_encode_plus,      a_,      max_length=a_,      padding="""max_length""",      )


 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  CodeGenTokenizer.from_pretrained(self.tmpdirname,      pad_token="""<pad>"""		)

   # Simple input
   _snake_case							:		List[Any]								=  """This is a simple input"""
   _snake_case							:		int								=  ["""This is a simple input looooooooong""", """This is a simple input"""]
   _snake_case							:		Any								=  ("""This is a simple input""", """This is a pair""")
   _snake_case							:		str								=  [
       ("""This is a simple input loooooong""", """This is a simple input"""),
       ("""This is a simple pair loooooong""", """This is a simple pair"""),
   ]

   _snake_case							:		str								=  tokenizer.pad_token_id

   _snake_case							:		Optional[int]								=  tokenizer(a_,      padding="""max_length""",      max_length=30,      return_tensors="""np"""		)
   _snake_case							:		Dict								=  tokenizer(a_,      padding=a_,      truncate=a_,      return_tensors="""np"""		)
   _snake_case							:		Tuple								=  tokenizer(*a_,      padding="""max_length""",      max_length=60,      return_tensors="""np"""		)
   _snake_case							:		Optional[Any]								=  tokenizer(a_,      padding=a_,      truncate=a_,      return_tensors="""np"""		)

   # s
   # test single string max_length padding
   self.assertEqual(out_s["""input_ids"""].shape[-1],      30		)
   self.assertTrue(pad_token_id in out_s["""input_ids"""]		)
   self.assertTrue(0 in out_s["""attention_mask"""]		)

   # s2
   # test automatic padding
   self.assertEqual(out_sa["""input_ids"""].shape[-1],      33		)
   # long slice doesn't have padding
   self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]		)
   self.assertFalse(0 in out_sa["""attention_mask"""][0]		)
   # short slice does have padding
   self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]		)
   self.assertTrue(0 in out_sa["""attention_mask"""][1]		)

   # p
   # test single pair max_length padding
   self.assertEqual(out_p["""input_ids"""].shape[-1],      60		)
   self.assertTrue(pad_token_id in out_p["""input_ids"""]		)
   self.assertTrue(0 in out_p["""attention_mask"""]		)

   # p2
   # test automatic padding pair
   self.assertEqual(out_pa["""input_ids"""].shape[-1],      52		)
   # long slice pair doesn't have padding
   self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]		)
   self.assertFalse(0 in out_pa["""attention_mask"""][0]		)
   # short slice pair does have padding
   self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]		)
   self.assertTrue(0 in out_pa["""attention_mask"""][1]		)


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Tuple								=  """$$$"""
   _snake_case							:		List[Any]								=  CodeGenTokenizer.from_pretrained(self.tmpdirname,      bos_token=a_,      add_bos_token=a_		)

   _snake_case							:		str								=  """This is a simple input"""
   _snake_case							:		int								=  ["""This is a simple input 1""", """This is a simple input 2"""]

   _snake_case							:		Union[str, Any]								=  tokenizer.bos_token_id

   _snake_case							:		Tuple								=  tokenizer(a_		)
   _snake_case							:		Optional[Any]								=  tokenizer(a_		)

   self.assertEqual(out_s.input_ids[0],      a_		)
   self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids		)		)

   _snake_case							:		Optional[int]								=  tokenizer.decode(out_s.input_ids		)
   _snake_case							:		int								=  tokenizer.batch_decode(out_sa.input_ids		)

   self.assertEqual(decode_s.split()[0],      a_		)
   self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa		)		)


 @slow
 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""		)

   _snake_case							:		Dict								=  """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
   _snake_case							:		Union[str, Any]								=  """\nif len_a > len_b:      result = a\nelse:      result = b"""

   _snake_case							:		Optional[Any]								=  tokenizer.encode(a_		)
   _snake_case							:		Dict								=  ["""^#""", re.escape("""<|endoftext|>"""		), """^'''""", """^\"\"\"""", """\n\n\n"""]
   _snake_case							:		Optional[Any]								=  tokenizer.decode(a_,      truncate_before_pattern=a_		)
   self.assertEqual(a_,      a_		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   pass








"""simple docstring"""


import argparse
import os

# New Code #
import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils import find_executable_batch_size


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to ensure out-of-memory errors never
# interrupt training, and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A_						=				16
A_						=				32


def 					UpperCAmelCase__       (snake_case__							:					Accelerator		,   snake_case__							:					int = 16    ):
  """simple docstring"""

  _snake_case							:		Optional[Any]								=  AutoTokenizer.from_pretrained("""bert-base-cased"""    )
  _snake_case							:		Any								=  load_dataset("""glue"""		,   """mrpc"""    )

  def tokenize_function(snake_case__							:					Any    ):
    # max_length=None => use the model max length (it's actually the default)
    _snake_case							:		Any								=  tokenizer(examples["""sentence1"""]		,   examples["""sentence2"""]		,   truncation=snake_case__		,   max_length=snake_case__    )
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    _snake_case							:		List[Any]								=  datasets.map(
        snake_case__		,   batched=snake_case__		,   remove_columns=["""idx""", """sentence1""", """sentence2"""]		,   )

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  _snake_case							:		int								=  tokenized_datasets.rename_column("""label"""		,   """labels"""    )

  def collate_fn(snake_case__							:					int    ):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    _snake_case							:		Optional[int]								=  1_28 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      _snake_case							:		str								=  16
    elif accelerator.mixed_precision != "no":
      _snake_case							:		Optional[int]								=  8
    else:
      _snake_case							:		Optional[int]								=  None

    return tokenizer.pad(
        snake_case__		,   padding="""longest"""		,   max_length=snake_case__		,   pad_to_multiple_of=snake_case__		,   return_tensors="""pt"""		,   )

  # Instantiate dataloaders.
  _snake_case							:		Optional[int]								=  DataLoader(
      tokenized_datasets["""train"""]		,   shuffle=snake_case__		,   collate_fn=snake_case__		,   batch_size=snake_case__    )
  _snake_case							:		Dict								=  DataLoader(
      tokenized_datasets["""validation"""]		,   shuffle=snake_case__		,   collate_fn=snake_case__		,   batch_size=snake_case__    )

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
     from accelerate.test_utils.training import mocked_dataloaders

     A_						=				mocked_dataloaders  # noqa: F811


def 					UpperCAmelCase__       (snake_case__							:					List[Any]		,   snake_case__							:					Any    ):
  """simple docstring"""

  if os.environ.get("""TESTING_MOCKED_DATALOADERS"""		,   snake_case__    ) == "1":
    _snake_case							:		List[Any]								=  2
  # Initialize accelerator
  _snake_case							:		str								=  Accelerator(cpu=args.cpu		,   mixed_precision=args.mixed_precision    )
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  _snake_case							:		Tuple								=  config["""lr"""]
  _snake_case							:		str								=  int(config["""num_epochs"""]    )
  _snake_case							:		Union[str, Any]								=  int(config["""seed"""]    )
  _snake_case							:		Union[str, Any]								=  int(config["""batch_size"""]    )

  _snake_case							:		List[str]								=  evaluate.load("""glue"""		,   """mrpc"""    )

  # New Code #
  # We now can define an inner training loop function. It should take a batch size as the only parameter,
  # and build the dataloaders in there.
  # It also gets our decorator
  @find_executable_batch_size(starting_batch_size=snake_case__    )
  def inner_training_loop(snake_case__							:					Union[str, Any]    ):
    # And now just move everything below under this function
    # We need to bring in the Accelerator object from earlier
    nonlocal accelerator
    # And reset all of its attributes that could hold onto any memory:
    accelerator.free_memory()

    # Then we can declare the model, optimizer, and everything else:
    set_seed(snake_case__    )

    # Instantiate the model (we build the model here so that the seed also control new weights initialization)
    _snake_case							:		List[Any]								=  AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""		,   return_dict=snake_case__    )

    # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
    # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
    # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
    _snake_case							:		Tuple								=  model.to(accelerator.device    )

    # Instantiate optimizer
    _snake_case							:		str								=  AdamW(params=model.parameters()		,   lr=snake_case__    )
    _snake_case		,					_snake_case							:		Optional[int]								=  get_dataloaders(snake_case__		,   snake_case__    )

    # Instantiate scheduler
    _snake_case							:		str								=  get_linear_schedule_with_warmup(
        optimizer=snake_case__		,   num_warmup_steps=1_00		,   num_training_steps=(len(snake_case__    ) * num_epochs)		,   )

    # Prepare everything
    # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
    # prepare method.
    _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		List[str]								=  accelerator.prepare(
        snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   snake_case__    )

    # Now we train the model
    for epoch in range(snake_case__    ):
      model.train()
      for step, batch in enumerate(snake_case__    ):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device    )
        _snake_case							:		int								=  model(**snake_case__    )
        _snake_case							:		str								=  outputs.loss
        accelerator.backward(snake_case__    )
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

      model.eval()
      for step, batch in enumerate(snake_case__    ):
        # We could avoid this line since we set the accelerator with `device_placement=True`.
        batch.to(accelerator.device    )
        with torch.no_grad():
          _snake_case							:		int								=  model(**snake_case__    )
        _snake_case							:		Optional[Any]								=  outputs.logits.argmax(dim=-1    )
        _snake_case		,					_snake_case							:		Tuple								=  accelerator.gather_for_metrics((predictions, batch["""labels"""])    )
        metric.add_batch(
            predictions=snake_case__		,   references=snake_case__		,   )

      _snake_case							:		str								=  metric.compute()
      # Use accelerator.print to print only on the main process.
      accelerator.print(F"epoch {epoch}:"		,   snake_case__    )

    # New Code #
    # And call it at the end with no arguments
    # Note: You could also refactor this outside of your training loop function
  inner_training_loop()


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Any								=  argparse.ArgumentParser(description="""Simple example of training script."""    )
  parser.add_argument(
      """--mixed_precision"""		,   type=snake_case__		,   default=snake_case__		,   choices=["""no""", """fp16""", """bf16""", """fp8"""]		,   help="""Whether to use mixed precision. Choose"""
      """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
      """and an Nvidia Ampere GPU."""		,   )
  parser.add_argument("""--cpu"""		,   action="""store_true"""		,   help="""If passed, will train on the CPU."""    )
  _snake_case							:		Dict								=  parser.parse_args()
  _snake_case							:		int								=  {"""lr""": 2e-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
  training_function(snake_case__		,   snake_case__    )


if __name__ == "__main__":
     main()


