
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


_UpperCAmelCase    			=					logging.get_logger(__name__)

_UpperCAmelCase    			=					{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_UpperCAmelCase    			=					{
    "vocab_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json",
    },
    "merges_file": {
        "openai/clip-vit-base-patch32": "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "openai/clip-vit-base-patch32": (
            "https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"
        ),
    },
}

_UpperCAmelCase    			=					{
    "openai/clip-vit-base-patch32": 77,
}

class 		__magic_name__				( lowercase_					):


  """simple docstring"""





  _UpperCamelCase						 = VOCAB_FILES_NAMES
  _UpperCamelCase						 = PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase						 = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCamelCase						 = ["input_ids", "attention_mask"]
  _UpperCamelCase						 = CLIPTokenizer


  def __init__(					self	,		a__=None	,		a__=None	,		a__=None	,		a__="<|endoftext|>"	,		a__="<|startoftext|>"	,		a__="<|endoftext|>"	,		a__="<|endoftext|>"	,		**a__	,		):
    super().__init__(
        a__	,		a__	,		tokenizer_file=a__	,		unk_token=a__	,		bos_token=a__	,		eos_token=a__	,		pad_token=a__	,		**a__	,		)

    if not isinstance(self.backend_tokenizer.pre_tokenizer	,		pre_tokenizers.Sequence		):
      raise ValueError(
          '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
          ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
          ''' to be compatible with this version.The easiest way to do so is'''
          ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
          ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
          ''' transformers.'''		)

    self._wrap_decode_method_backend_tokenizer()


  def     _UpperCAmelCase  (					self		):
    _lowerCamelCase      =						self.backend_tokenizer.decode

    def new_decode_method(*a__	,		**a__		):
      _lowerCamelCase      =						orig_decode_method(*a__	,		**a__		)
      _lowerCamelCase      =						text.replace(self.backend_tokenizer.model.end_of_word_suffix	,		''' '''		).strip()
      return text

    _lowerCamelCase      =						new_decode_method


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    _lowerCamelCase      =						[self.bos_token_id]
    _lowerCamelCase      =						[self.eos_token_id]

    if token_ids_a is None:
      return bos_token + token_ids_a + eos_token
    return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    _lowerCamelCase      =						[self.bos_token_id]
    _lowerCamelCase      =						[self.eos_token_id]

    if token_ids_a is None:
      return len(bos_token + token_ids_a + eos_token		) * [0]
    return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token		) * [0]


  def     _UpperCAmelCase  (					self	,		a__	,		a__ = None		):
    _lowerCamelCase      =						self._tokenizer.model.save(a__	,		name=a__		)
    return tuple(a__		)






from __future__ import annotations

from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any

class 		__magic_name__				:


  """simple docstring"""





  _UpperCamelCase						 = ("_front", "_back", "_len")

  @dataclass
  class 		__magic_name__				:


    """simple docstring"""





    _UpperCamelCase						 = None
    _UpperCamelCase						 = None
    _UpperCamelCase						 = None







  class 		__magic_name__				:


    """simple docstring"""





    _UpperCamelCase						 = ("_cur",)


    def __init__(					self	,		a__		):
      _lowerCamelCase      =						cur


    def __iter__(					self		):
      return self


    def     _UpperCAmelCase  (					self		):
      if self._cur is None:
        # finished iterating
        raise StopIteration
      _lowerCamelCase      =						self._cur.val
      _lowerCamelCase      =						self._cur.next_node

      return val







  def __init__(					self	,		a__ = None		):
    _lowerCamelCase      =						None
    _lowerCamelCase      =						None
    _lowerCamelCase      =						0

    if iterable is not None:
      # append every value to the deque
      for val in iterable:
        self.append(a__		)


  def     _UpperCAmelCase  (					self	,		a__		):
    _lowerCamelCase      =						self._Node(a__	,		a__	,		a__		)
    if self.is_empty():
      # front = back
      _lowerCamelCase      =						_lowerCamelCase      =						node
      _lowerCamelCase      =						1
    else:
      # connect nodes
      _lowerCamelCase      =						node
      _lowerCamelCase      =						self._back
      _lowerCamelCase      =						node  # assign new back to the new node

      self._len += 1

      # make sure there were no errors
      assert not self.is_empty(), "Error on appending value."


  def     _UpperCAmelCase  (					self	,		a__		):
    _lowerCamelCase      =						self._Node(a__	,		a__	,		a__		)
    if self.is_empty():
      # front = back
      _lowerCamelCase      =						_lowerCamelCase      =						node
      _lowerCamelCase      =						1
    else:
      # connect nodes
      _lowerCamelCase      =						self._front
      _lowerCamelCase      =						node
      _lowerCamelCase      =						node  # assign new front to the new node

      self._len += 1

      # make sure there were no errors
      assert not self.is_empty(), "Error on appending value."


  def     _UpperCAmelCase  (					self	,		a__		):
    for val in iterable:
      self.append(a__		)


  def     _UpperCAmelCase  (					self	,		a__		):
    for val in iterable:
      self.appendleft(a__		)


  def     _UpperCAmelCase  (					self		):
    assert not self.is_empty(), "Deque is empty."

    _lowerCamelCase      =						self._back
    _lowerCamelCase      =						self._back.prev_node  # set new back
    # drop the last node - python will deallocate memory automatically
    _lowerCamelCase      =						None

    self._len -= 1

    return topop.val


  def     _UpperCAmelCase  (					self		):
    assert not self.is_empty(), "Deque is empty."

    _lowerCamelCase      =						self._front
    _lowerCamelCase      =						self._front.next_node  # set new front and drop the first node
    _lowerCamelCase      =						None

    self._len -= 1

    return topop.val


  def     _UpperCAmelCase  (					self		):
    return self._front is None


  def __len__(					self		):
    return self._len


  def __eq__(					self	,		a__		):

    if not isinstance(a__	,		a__		):
      return NotImplemented

    _lowerCamelCase      =						self._front
    _lowerCamelCase      =						other._front

    # if the length of the dequeues are not the same, they are not equal
    if len(self		) != len(a__		):
      return False

    while me is not None and oth is not None:
      # compare every value
      if me.val != oth.val:
        return False
      _lowerCamelCase      =						me.next_node
      _lowerCamelCase      =						oth.next_node

    return True


  def __iter__(					self		):
    return Deque._Iterator(self._front		)


  def __repr__(					self		):
    _lowerCamelCase      =						[]
    _lowerCamelCase      =						self._front
    while aux is not None:
      # append the values in a list to display
      values_list.append(aux.val		)
      _lowerCamelCase      =						aux.next_node

    return f'''[{', '.join(repr(a__		) for val in values_list		)}]'''


if __name__ == "__main__":
  import doctest

  doctest.testmod()




