'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

lowerCAmelCase_			:			Dict    	=   logging.get_logger(__name__)

lowerCAmelCase_			:			int    	=   {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

lowerCAmelCase_			:			Optional[Any]    	=   {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

lowerCAmelCase_			:			Any    	=   {
    "gpt2": 1_024,
    "gpt2-medium": 1_024,
    "gpt2-large": 1_024,
    "gpt2-xl": 1_024,
    "distilgpt2": 1_024,
}


@lru_cache()
def __a			(						)						->				Any:
      '''simple docstring'''


      lowercase_     			=   (
          list(range(ord("!"					)			,		ord("~"					) + 1					)					) + list(range(ord("¡"					)			,		ord("¬"					) + 1					)					) + list(range(ord("®"					)			,		ord("ÿ"					) + 1					)					)
      )
      lowercase_     			=   bs[:]
      lowercase_     			=   0
      for b in range(2**8					):
            if b not in bs:
                  bs.append(__lowerCamelCase					)
                  cs.append(2**8 + n					)
                  n += 1
      lowercase_     			=   [chr(__lowerCamelCase					) for n in cs]
      return dict(zip(__lowerCamelCase			,		__lowerCamelCase					)					)


def __a			(						__lowerCamelCase      :    Optional[int]					)						->				Tuple:
      '''simple docstring'''


      lowercase_     			=   set()
      lowercase_     			=   word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)					)
            lowercase_     			=   char
      return pairs






class 				lowercase    (    __lowerCamelCase			):
   lowerCamelCase_            =VOCAB_FILES_NAMES
   lowerCamelCase_            =PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase_            =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase_            =['input_ids', 'attention_mask']


   def __init__( self					:						Optional[int] ,		__lowerCAmelCase					:						str ,		__lowerCAmelCase					:						Tuple ,		__lowerCAmelCase					:						Optional[int]="replace" ,		__lowerCAmelCase					:						List[Any]="<|endoftext|>" ,		__lowerCAmelCase					:						Tuple="<|endoftext|>" ,		__lowerCAmelCase					:						Dict="<|endoftext|>" ,		__lowerCAmelCase					:						Tuple=None ,		__lowerCAmelCase					:						Any=False ,		__lowerCAmelCase					:						int=False ,		**__lowerCAmelCase					:						Any ,		)   ->       Tuple:
         lowercase_     			=   AddedToken(__lowerCAmelCase ,		lstrip=__lowerCAmelCase ,		rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase ,		__lowerCAmelCase) else bos_token
         lowercase_     			=   AddedToken(__lowerCAmelCase ,		lstrip=__lowerCAmelCase ,		rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase ,		__lowerCAmelCase) else eos_token
         lowercase_     			=   AddedToken(__lowerCAmelCase ,		lstrip=__lowerCAmelCase ,		rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase ,		__lowerCAmelCase) else unk_token
         lowercase_     			=   AddedToken(__lowerCAmelCase ,		lstrip=__lowerCAmelCase ,		rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase ,		__lowerCAmelCase) else pad_token
         super().__init__(
             errors=__lowerCAmelCase ,		unk_token=__lowerCAmelCase ,		bos_token=__lowerCAmelCase ,		eos_token=__lowerCAmelCase ,		pad_token=__lowerCAmelCase ,		add_prefix_space=__lowerCAmelCase ,		add_bos_token=__lowerCAmelCase ,		**__lowerCAmelCase ,		)
         lowercase_     			=   add_bos_token

         with open(__lowerCAmelCase ,		encoding="utf-8") as vocab_handle:
               lowercase_     			=   json.load(__lowerCAmelCase)
         lowercase_     			=   {v: k for k, v in self.encoder.items()}
         lowercase_     			=   errors  # how to handle errors in decoding
         lowercase_     			=   bytes_to_unicode()
         lowercase_     			=   {v: k for k, v in self.byte_encoder.items()}
         with open(__lowerCAmelCase ,		encoding="utf-8") as merges_handle:
               lowercase_     			=   merges_handle.read().split("\n")[1:-1]
         lowercase_     			=   [tuple(merge.split()) for merge in bpe_merges]
         lowercase_     			=   dict(zip(__lowerCAmelCase ,		range(len(__lowerCAmelCase))))
         lowercase_     			=   {}
         lowercase_     			=   add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowercase_     			=   re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")


   @property
   def __UpperCAmelCase							( self					:						List[Any])   ->       int:
         return len(self.encoder)


   def __UpperCAmelCase							( self					:						List[Any])   ->       int:
         return dict(self.encoder ,		**self.added_tokens_encoder)


   def __UpperCAmelCase							( self					:						Union[str, Any] ,		__lowerCAmelCase					:						int)   ->       Union[str, Any]:
         if token in self.cache:
               return self.cache[token]
         lowercase_     			=   tuple(__lowerCAmelCase)
         lowercase_     			=   get_pairs(__lowerCAmelCase)

         if not pairs:
               return token

         while True:
               lowercase_     			=   min(__lowerCAmelCase ,		key=lambda __lowerCAmelCase: self.bpe_ranks.get(__lowerCAmelCase ,		float("inf")))
               if bigram not in self.bpe_ranks:
                     break
               lowercase_			,				lowercase_     			=   bigram
               lowercase_     			=   []
               lowercase_     			=   0
               while i < len(__lowerCAmelCase):
                     try:
                           lowercase_     			=   word.index(__lowerCAmelCase ,		__lowerCAmelCase)
                     except ValueError:
                           new_word.extend(word[i:])
                           break
                     else:
                           new_word.extend(word[i:j])
                           lowercase_     			=   j

                     if word[i] == first and i < len(__lowerCAmelCase) - 1 and word[i + 1] == second:
                           new_word.append(first + second)
                           i += 2
                     else:
                           new_word.append(word[i])
                           i += 1
               lowercase_     			=   tuple(__lowerCAmelCase)
               lowercase_     			=   new_word
               if len(__lowerCAmelCase) == 1:
                     break
               else:
                     lowercase_     			=   get_pairs(__lowerCAmelCase)
         lowercase_     			=   " ".join(__lowerCAmelCase)
         lowercase_     			=   word
         return word


   def __UpperCAmelCase							( self					:						Optional[Any] ,		__lowerCAmelCase					:						Dict ,		__lowerCAmelCase					:						Optional[Any]=None)   ->       Dict:
         if self.add_bos_token:
               lowercase_     			=   [self.bos_token_id]
         else:
               lowercase_     			=   []

         lowercase_     			=   bos_token_ids + token_ids_a

         if token_ids_a is None:
               return output

         return output + bos_token_ids + token_ids_a


   def __UpperCAmelCase							( self					:						Any ,		__lowerCAmelCase					:						List[int] ,		__lowerCAmelCase					:						Optional[List[int]] = None ,		__lowerCAmelCase					:						bool = False)   ->       List[int]:
         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__lowerCAmelCase ,		token_ids_a=__lowerCAmelCase ,		already_has_special_tokens=__lowerCAmelCase)

         if not self.add_bos_token:
               return super().get_special_tokens_mask(
                   token_ids_a=__lowerCAmelCase ,		token_ids_a=__lowerCAmelCase ,		already_has_special_tokens=__lowerCAmelCase)

         if token_ids_a is None:
               return [1] + ([0] * len(__lowerCAmelCase))
         return [1] + ([0] * len(__lowerCAmelCase)) + [1] + ([0] * len(__lowerCAmelCase))


   def __UpperCAmelCase							( self					:						Optional[int] ,		__lowerCAmelCase					:						Optional[int])   ->       int:
         lowercase_     			=   []
         for token in re.findall(self.pat ,		__lowerCAmelCase):
               lowercase_     			=   "".join(
                   self.byte_encoder[b] for b in token.encode("utf-8"))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase).split(" "))
         return bpe_tokens


   def __UpperCAmelCase							( self					:						Tuple ,		__lowerCAmelCase					:						Union[str, Any])   ->       str:
         return self.encoder.get(__lowerCAmelCase ,		self.encoder.get(self.unk_token))


   def __UpperCAmelCase							( self					:						Any ,		__lowerCAmelCase					:						List[str])   ->       Optional[Any]:
         return self.decoder.get(__lowerCAmelCase)


   def __UpperCAmelCase							( self					:						str ,		__lowerCAmelCase					:						Dict)   ->       str:
         lowercase_     			=   "".join(__lowerCAmelCase)
         lowercase_     			=   bytearray([self.byte_decoder[c] for c in text]).decode("utf-8" ,		errors=self.errors)
         return text


   def __UpperCAmelCase							( self					:						Union[str, Any] ,		__lowerCAmelCase					:						str ,		__lowerCAmelCase					:						Optional[str] = None)   ->       Tuple[str]:
         if not os.path.isdir(__lowerCAmelCase):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
         lowercase_     			=   os.path.join(
             __lowerCAmelCase ,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
         lowercase_     			=   os.path.join(
             __lowerCAmelCase ,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

         with open(__lowerCAmelCase ,		"w" ,		encoding="utf-8") as f:
               f.write(json.dumps(self.encoder ,		indent=2 ,		sort_keys=__lowerCAmelCase ,		ensure_ascii=__lowerCAmelCase) + "\n")

         lowercase_     			=   0
         with open(__lowerCAmelCase ,		"w" ,		encoding="utf-8") as writer:
               writer.write("#version: 0.2\n")
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,		key=lambda __lowerCAmelCase: kv[1]):
                     if index != token_index:
                           logger.warning(
                               F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                               " Please check that the tokenizer is not corrupted!")
                           lowercase_     			=   token_index
                     writer.write(" ".join(__lowerCAmelCase) + "\n")
                     index += 1

         return vocab_file, merge_file


   def __UpperCAmelCase							( self					:						Tuple ,		__lowerCAmelCase					:						Optional[int] ,		__lowerCAmelCase					:						Any=False ,		**__lowerCAmelCase					:						int)   ->       str:
         lowercase_     			=   kwargs.pop("add_prefix_space" ,		self.add_prefix_space)
         if is_split_into_words or add_prefix_space:
               lowercase_     			=   " " + text
         return (text, kwargs)


   def __UpperCAmelCase							( self					:						Optional[Any] ,		__lowerCAmelCase					:						"Conversation")   ->       List[int]:
         lowercase_     			=   []
         for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(__lowerCAmelCase ,		add_special_tokens=__lowerCAmelCase) + [self.eos_token_id])
         if len(__lowerCAmelCase) > self.model_max_length:
               lowercase_     			=   input_ids[-self.model_max_length :]
         return input_ids







'''simple docstring'''

import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import ConvNextImageProcessor





class 				lowercase    (    unittest.TestCase			):


   def __init__( self					:						str ,		__lowerCAmelCase					:						Union[str, Any] ,		__lowerCAmelCase					:						Optional[int]=7 ,		__lowerCAmelCase					:						Optional[Any]=3 ,		__lowerCAmelCase					:						Optional[Any]=18 ,		__lowerCAmelCase					:						List[str]=30 ,		__lowerCAmelCase					:						List[str]=400 ,		__lowerCAmelCase					:						Tuple=True ,		__lowerCAmelCase					:						Optional[int]=None ,		__lowerCAmelCase					:						List[str]=0.875 ,		__lowerCAmelCase					:						int=True ,		__lowerCAmelCase					:						Optional[int]=[0.5, 0.5, 0.5] ,		__lowerCAmelCase					:						List[str]=[0.5, 0.5, 0.5] ,		)   ->       Optional[Any]:
         lowercase_     			=   size if size is not None else {"shortest_edge": 20}
         lowercase_     			=   parent
         lowercase_     			=   batch_size
         lowercase_     			=   num_channels
         lowercase_     			=   image_size
         lowercase_     			=   min_resolution
         lowercase_     			=   max_resolution
         lowercase_     			=   do_resize
         lowercase_     			=   size
         lowercase_     			=   crop_pct
         lowercase_     			=   do_normalize
         lowercase_     			=   image_mean
         lowercase_     			=   image_std


   def __UpperCAmelCase							( self					:						List[str])   ->       int:
         return {
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_normalize": self.do_normalize,
             "do_resize": self.do_resize,
             "size": self.size,
             "crop_pct": self.crop_pct,
         }





@require_torch
@require_vision
class 				lowercase    (    __lowerCamelCase			,					unittest.TestCase			):
   lowerCamelCase_            =ConvNextImageProcessor if is_vision_available() else None


   def __UpperCAmelCase							( self					:						Optional[Any])   ->       Tuple:
         lowercase_     			=   ConvNextImageProcessingTester(self)


   @property
   def __UpperCAmelCase							( self					:						Optional[int])   ->       int:
         return self.image_processor_tester.prepare_image_processor_dict()


   def __UpperCAmelCase							( self					:						Optional[int])   ->       Any:
         lowercase_     			=   self.image_processing_class(**self.image_processor_dict)
         self.assertTrue(hasattr(__lowerCAmelCase ,		"do_resize"))
         self.assertTrue(hasattr(__lowerCAmelCase ,		"size"))
         self.assertTrue(hasattr(__lowerCAmelCase ,		"crop_pct"))
         self.assertTrue(hasattr(__lowerCAmelCase ,		"do_normalize"))
         self.assertTrue(hasattr(__lowerCAmelCase ,		"image_mean"))
         self.assertTrue(hasattr(__lowerCAmelCase ,		"image_std"))


   def __UpperCAmelCase							( self					:						List[str])   ->       Any:
         lowercase_     			=   self.image_processing_class.from_dict(self.image_processor_dict)
         self.assertEqual(image_processor.size ,		{"shortest_edge": 20})

         lowercase_     			=   self.image_processing_class.from_dict(self.image_processor_dict ,		size=42)
         self.assertEqual(image_processor.size ,		{"shortest_edge": 42})


   def __UpperCAmelCase							( self					:						Any)   ->       Optional[int]:
         pass


   def __UpperCAmelCase							( self					:						List[Any])   ->       Any:
         # Initialize image_processing
         lowercase_     			=   self.image_processing_class(**self.image_processor_dict)
         # create random PIL images
         lowercase_     			=   prepare_image_inputs(self.image_processor_tester ,		equal_resolution=__lowerCAmelCase)
         for image in image_inputs:
               self.assertIsInstance(__lowerCAmelCase ,		Image.Image)

         # Test not batched input
         lowercase_     			=   image_processing(image_inputs[0] ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)

         # Test batched
         lowercase_     			=   image_processing(__lowerCAmelCase ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)


   def __UpperCAmelCase							( self					:						Union[str, Any])   ->       str:
         # Initialize image_processing
         lowercase_     			=   self.image_processing_class(**self.image_processor_dict)
         # create random numpy tensors
         lowercase_     			=   prepare_image_inputs(self.image_processor_tester ,		equal_resolution=__lowerCAmelCase ,		numpify=__lowerCAmelCase)
         for image in image_inputs:
               self.assertIsInstance(__lowerCAmelCase ,		np.ndarray)

         # Test not batched input
         lowercase_     			=   image_processing(image_inputs[0] ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)

         # Test batched
         lowercase_     			=   image_processing(__lowerCAmelCase ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)


   def __UpperCAmelCase							( self					:						List[str])   ->       List[str]:
         # Initialize image_processing
         lowercase_     			=   self.image_processing_class(**self.image_processor_dict)
         # create random PyTorch tensors
         lowercase_     			=   prepare_image_inputs(self.image_processor_tester ,		equal_resolution=__lowerCAmelCase ,		torchify=__lowerCAmelCase)
         for image in image_inputs:
               self.assertIsInstance(__lowerCAmelCase ,		torch.Tensor)

         # Test not batched input
         lowercase_     			=   image_processing(image_inputs[0] ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)

         # Test batched
         lowercase_     			=   image_processing(__lowerCAmelCase ,		return_tensors="pt").pixel_values
         self.assertEqual(
             encoded_images.shape ,		(
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.size["shortest_edge"],
                 self.image_processor_tester.size["shortest_edge"],
             ) ,		)






