


'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class      snake_case    (    lowercase			,						unittest.TestCase			):





  """simple docstring"""




  _lowerCamelCase		    =  CLIPTokenizer
  _lowerCamelCase		    =  CLIPTokenizerFast
  _lowerCamelCase		    =  True
  _lowerCamelCase		    =  {}
  _lowerCamelCase		    =  False






  def   snake_case		(    self     ):







   """simple docstring"""
   super().setUp()

   # fmt: off
   lowerCamelCase_      =  ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
   # fmt: on
   lowerCamelCase_      =  dict(zip(UpperCamelCase			,  range(len(UpperCamelCase     )     )     )     )
   lowerCamelCase_      =  ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
   lowerCamelCase_      =  {"unk_token": "<unk>"}

   lowerCamelCase_      =  os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["vocab_file"]     )
   lowerCamelCase_      =  os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["merges_file"]     )
   with open(self.vocab_file			,  "w"			,  encoding="utf-8"     ) as fp:
    fp.write(json.dumps(UpperCamelCase     ) + "\n"     )
   with open(self.merges_file			,  "w"			,  encoding="utf-8"     ) as fp:
    fp.write("\n".join(UpperCamelCase     )     )






  def   snake_case		(    self			,  **UpperCamelCase     ):







   """simple docstring"""
   kwargs.update(self.special_tokens_map     )
   return CLIPTokenizer.from_pretrained(self.tmpdirname			,  **UpperCamelCase     )






  def   snake_case		(    self			,  **UpperCamelCase     ):







   """simple docstring"""
   kwargs.update(self.special_tokens_map     )
   return CLIPTokenizerFast.from_pretrained(self.tmpdirname			,  **UpperCamelCase     )






  def   snake_case		(    self			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  "lower newer"
   lowerCamelCase_      =  "lower newer"
   return input_text, output_text






  def   snake_case		(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  CLIPTokenizer(self.vocab_file			,  self.merges_file			,  **self.special_tokens_map     )
   lowerCamelCase_      =  "lower newer"
   lowerCamelCase_      =  ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
   lowerCamelCase_      =  tokenizer.tokenize(UpperCamelCase     )
   self.assertListEqual(UpperCamelCase			,  UpperCamelCase     )

   lowerCamelCase_      =  tokens + [tokenizer.unk_token]
   lowerCamelCase_      =  [10, 2, 16, 9, 3, 2, 16, 20]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase     )			,  UpperCamelCase     )






  @require_ftfy
  def   snake_case		(    self     ):







   """simple docstring"""
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
     lowerCamelCase_      =  self.tokenizer_class.from_pretrained(UpperCamelCase			,  **UpperCamelCase     )
     lowerCamelCase_      =  self.rust_tokenizer_class.from_pretrained(UpperCamelCase			,  **UpperCamelCase     )

     lowerCamelCase_      =  "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
     lowerCamelCase_      =  tokenizer_s.tokenize(UpperCamelCase     )
     lowerCamelCase_      =  tokenizer_r.tokenize(UpperCamelCase     )

     self.assertListEqual(UpperCamelCase			,  UpperCamelCase     )

     # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
     # with Tilde) encoded in 2 different ways
     lowerCamelCase_      =  "xa\u0303y" + " " + "x\xe3y"
     lowerCamelCase_      =  tokenizer_s.tokenize(UpperCamelCase     )
     lowerCamelCase_      =  tokenizer_r.tokenize(UpperCamelCase     )

     self.assertListEqual(UpperCamelCase			,  UpperCamelCase     )

     # Test that the tokenization is identical on unicode of space type
     lowerCamelCase_      =  [
         "\u0009",  # (horizontal tab, '\t')
         "\u000B",  # (vertical tab)
         "\u000C",  # (form feed)
         "\u0020",  # (space, ' ')
         "\u200E",  # (left-to-right mark):w
         "\u200F",  # (right-to-left mark)
     ]
     for unicode_seq in spaces_unicodes:
      lowerCamelCase_      =  tokenizer_s.tokenize(UpperCamelCase     )
      lowerCamelCase_      =  tokenizer_r.tokenize(UpperCamelCase     )

      self.assertListEqual(UpperCamelCase			,  UpperCamelCase     )

     # Test that the tokenization is identical on unicode of line break type
     lowerCamelCase_      =  [
         "\u000A",  # (line feed, '\n')
         "\r\n",  # (carriage return and line feed, '\r\n')
         "\u000D",  # (carriage return, '\r')
         "\r",  # (carriage return, '\r')
         "\u000D",  # (carriage return, '\r')
         "\u2028",  # (line separator)
         "\u2029",  # (paragraph separator)
         # "\u0085", # (next line)
     ]

     # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
     # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
     # space (and thus into an empty list).

     for unicode_seq in line_break_unicodes:
      lowerCamelCase_      =  tokenizer_s.tokenize(UpperCamelCase     )
      lowerCamelCase_      =  tokenizer_r.tokenize(UpperCamelCase     )

      self.assertListEqual(UpperCamelCase			,  UpperCamelCase     )






  def   snake_case		(    self     ):







   """simple docstring"""
   # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
     lowerCamelCase_      =  "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
     lowerCamelCase_      =  f'''{text_of_1_token} {text_of_1_token}'''

     lowerCamelCase_      =  self.rust_tokenizer_class.from_pretrained(
         UpperCamelCase			,  use_fast=UpperCamelCase			,  )
     lowerCamelCase_      =  tokenizer_r(UpperCamelCase			,  return_offsets_mapping=UpperCamelCase			,  add_special_tokens=UpperCamelCase     )
     self.assertEqual(encoding.offset_mapping[0]			,  (0, len(UpperCamelCase     ))     )
     self.assertEqual(
         encoding.offset_mapping[1]			,  (len(UpperCamelCase     ) + 1, len(UpperCamelCase     ) + 1 + len(UpperCamelCase     ))			,  )

     lowerCamelCase_      =  f''' {text}'''

     lowerCamelCase_      =  self.rust_tokenizer_class.from_pretrained(
         UpperCamelCase			,  use_fast=UpperCamelCase			,  )
     lowerCamelCase_      =  tokenizer_r(UpperCamelCase			,  return_offsets_mapping=UpperCamelCase			,  add_special_tokens=UpperCamelCase     )
     self.assertEqual(encoding.offset_mapping[0]			,  (1, 1 + len(UpperCamelCase     ))     )
     self.assertEqual(
         encoding.offset_mapping[1]			,  (1 + len(UpperCamelCase     ) + 1, 1 + len(UpperCamelCase     ) + 1 + len(UpperCamelCase     ))			,  )






  def   snake_case		(    self     ):







   """simple docstring"""
   # Test related to the breaking change introduced in transformers v4.17.0
   # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
   with self.assertRaises(UpperCamelCase     ) as context:
    self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"     )

   self.assertTrue(
       context.exception.args[0].startswith(
           "The `backend_tokenizer` provided does not match the expected format."     )     )






  @require_ftfy
  def   snake_case		(    self     ):







   """simple docstring"""
   super().test_tokenization_python_rust_equals()







  def   snake_case		(    self     ):







   """simple docstring"""
   # CLIP always lower cases letters
   pass







'''simple docstring'''

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser





def __snake_case					(      ):
 lowerCamelCase_      =  ArgumentParser("Accelerate CLI tool"					,				usage="accelerate <command> [<args>]"					,				allow_abbrev=UpperCAmelCase_		)
 lowerCamelCase_      =  parser.add_subparsers(help="accelerate command helpers"		)

 # Register commands
 get_config_parser(subparsers=UpperCAmelCase_		)
 env_command_parser(subparsers=UpperCAmelCase_		)
 launch_command_parser(subparsers=UpperCAmelCase_		)
 tpu_command_parser(subparsers=UpperCAmelCase_		)
 test_command_parser(subparsers=UpperCAmelCase_		)

 # Let's go
 lowerCamelCase_      =  parser.parse_args()

 if not hasattr(UpperCAmelCase_					,				"func"		):
  parser.print_help()
  exit(1		)

 # Run
 args.func(UpperCAmelCase_		)


if __name__ == "__main__":
     main()



