




import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_lowerCAmelCase    : int		= logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_lowerCAmelCase    : Tuple		= direct_transformers_import(Path(__file__).parent)


_lowerCAmelCase    : List[str]		= {
    "AutoTokenizer": "PreTrainedTokenizerBase",
    "AutoFeatureExtractor": "FeatureExtractionMixin",
    "AutoImageProcessor": "ImageProcessingMixin",
}
class       _UpperCamelCase   (					lowerCAmelCase		):
      UpperCAmelCase_          =		["""feature_extractor""", """tokenizer"""]
      # Names need to be attr_class for attr in attributes
      UpperCAmelCase_          =		None
      UpperCAmelCase_          =		None
      UpperCAmelCase_          =		None
      def __init__(      self :Any			,   *lowerCamelCase :List[Any]			,   **lowerCamelCase :Optional[Any]   )  ->      Union[str, Any]:
       # Sanitize args and kwargs
       for key in kwargs:
        if key not in self.attributes:
         raise TypeError(f'''Unexpected keyword argument {key}.'''   )
       for arg, attribute_name in zip(lowerCamelCase			,   self.attributes   ):
        if attribute_name in kwargs:
         raise TypeError(f'''Got multiple values for argument {attribute_name}.'''   )
        else:
         UpperCAmelCase__							=    arg

       if len(lowerCamelCase   ) != len(self.attributes   ):
        raise ValueError(
            f'''This processor requires {len(self.attributes   )} arguments: {", ".join(self.attributes   )}. Got '''
            f'''{len(lowerCamelCase   )} arguments instead.'''   )

       # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
       for attribute_name, arg in kwargs.items():
        UpperCAmelCase__							=    getattr(self			,   f'''{attribute_name}_class'''   )
        # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
        UpperCAmelCase__							=    AUTO_TO_BASE_CLASS_MAPPING.get(lowerCamelCase			,   lowerCamelCase   )
        if isinstance(lowerCamelCase			,   lowerCamelCase   ):
         UpperCAmelCase__							=    tuple(getattr(lowerCamelCase			,   lowerCamelCase   ) for n in class_name if n is not None   )
        else:
         UpperCAmelCase__							=    getattr(lowerCamelCase			,   lowerCamelCase   )

        if not isinstance(lowerCamelCase			,   lowerCamelCase   ):
         raise ValueError(
             f'''Received a {type(lowerCamelCase   ).__name__} for argument {attribute_name}, but a {class_name} was expected.'''   )

        setattr(self			,   lowerCamelCase			,   lowerCamelCase   )
      def __repr__(      self :Optional[Any]   )  ->      Optional[Any]:
       UpperCAmelCase__							=    [f'''- {name}: {repr(getattr(self			,   lowerCamelCase   )   )}''' for name in self.attributes]
       UpperCAmelCase__							=    "\n".join(lowerCamelCase   )
       return f'''{self.__class__.__name__}:\n{attributes_repr}'''
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :int			,   lowerCamelCase :bool = False			,   **lowerCamelCase :Tuple   )  ->      List[str]:
       os.makedirs(lowerCamelCase			,   exist_ok=lowerCamelCase   )

       if push_to_hub:
        UpperCAmelCase__							=    kwargs.pop("commit_message"			,   lowerCamelCase   )
        UpperCAmelCase__							=    kwargs.pop("repo_id"			,   save_directory.split(os.path.sep   )[-1]   )
        UpperCAmelCase__							=    self._create_repo(lowerCamelCase			,   **lowerCamelCase   )
        UpperCAmelCase__							=    self._get_files_timestamps(lowerCamelCase   )
       # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
       # loaded from the Hub.
       if self._auto_class is not None:
        UpperCAmelCase__							=    [getattr(self			,   lowerCamelCase   ) for attribute_name in self.attributes]
        UpperCAmelCase__							=    [(a.init_kwargs if isinstance(lowerCamelCase			,   lowerCamelCase   ) else a) for a in attrs]
        custom_object_save(self			,   lowerCamelCase			,   config=lowerCamelCase   )

       for attribute_name in self.attributes:
        UpperCAmelCase__							=    getattr(self			,   lowerCamelCase   )
        # Include the processor class in the attribute config so this processor can then be reloaded with the
        # `AutoProcessor` API.
        if hasattr(lowerCamelCase			,   "_set_processor_class"   ):
         attribute._set_processor_class(self.__class__.__name__   )
        attribute.save_pretrained(lowerCamelCase   )

       if self._auto_class is not None:
        # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
        for attribute_name in self.attributes:
         UpperCAmelCase__							=    getattr(self			,   lowerCamelCase   )
         if isinstance(lowerCamelCase			,   lowerCamelCase   ):
          del attribute.init_kwargs["auto_map"]

       if push_to_hub:
        self._upload_modified_files(
            lowerCamelCase			,   lowerCamelCase			,   lowerCamelCase			,   commit_message=lowerCamelCase			,   token=kwargs.get("use_auth_token"   )			,   )
      @classmethod
      def 						UpperCAmelCase_    (      cls :str			,   lowerCamelCase :Union[str, os.PathLike]			,   lowerCamelCase :Optional[Union[str, os.PathLike]] = None			,   lowerCamelCase :bool = False			,   lowerCamelCase :bool = False			,   lowerCamelCase :Optional[Union[str, bool]] = None			,   lowerCamelCase :str = "main"			,   **lowerCamelCase :Any			,   )  ->      Union[str, Any]:
       UpperCAmelCase__							=    cache_dir
       UpperCAmelCase__							=    force_download
       UpperCAmelCase__							=    local_files_only
       UpperCAmelCase__							=    revision

       UpperCAmelCase__							=    kwargs.pop("use_auth_token"			,   lowerCamelCase   )
       if use_auth_token is not None:
        warnings.warn(
            "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."			,   lowerCamelCase   )
        if token is not None:
         raise ValueError(
             "`token` and `use_auth_token` are both specified. Please set only the argument `token`."   )
        UpperCAmelCase__							=    use_auth_token

       if token is not None:
        # change to `token` in a follow-up PR
        UpperCAmelCase__							=    token

       UpperCAmelCase__							=    cls._get_arguments_from_pretrained(lowerCamelCase			,   **lowerCamelCase   )
       return cls(*lowerCamelCase   )
      @classmethod
      def 						UpperCAmelCase_    (      cls :Optional[int]			,   lowerCamelCase :Union[str, Any]="AutoProcessor"   )  ->      Optional[int]:
       if not isinstance(lowerCamelCase			,   lowerCamelCase   ):
        UpperCAmelCase__							=    auto_class.__name__

       import transformers.models.auto as auto_module

       if not hasattr(lowerCamelCase			,   lowerCamelCase   ):
        raise ValueError(f'''{auto_class} is not a valid auto class.'''   )

       UpperCAmelCase__							=    auto_class
      @classmethod
      def 						UpperCAmelCase_    (      cls :int			,   lowerCamelCase :int			,   **lowerCamelCase :str   )  ->      Optional[int]:
       UpperCAmelCase__							=    []
       for attribute_name in cls.attributes:
        UpperCAmelCase__							=    getattr(cls			,   f'''{attribute_name}_class'''   )
        if isinstance(lowerCamelCase			,   lowerCamelCase   ):
         UpperCAmelCase__							=    tuple(getattr(lowerCamelCase			,   lowerCamelCase   ) if n is not None else None for n in class_name   )
         UpperCAmelCase__							=    kwargs.get("use_fast"			,   lowerCamelCase   )
         if use_fast and classes[1] is not None:
          UpperCAmelCase__							=    classes[1]
         else:
          UpperCAmelCase__							=    classes[0]
        else:
         UpperCAmelCase__							=    getattr(lowerCamelCase			,   lowerCamelCase   )

        args.append(attribute_class.from_pretrained(lowerCamelCase			,   **lowerCamelCase   )   )
       return args







      @property
      def 						UpperCAmelCase_    (      self :Optional[int]   )  ->      Any:
       UpperCAmelCase__							=    getattr(self			,   self.attributes[0]   )
       return getattr(lowerCamelCase			,   "model_input_names"			,   lowerCamelCase   )



_lowerCAmelCase    : Dict		= copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
  _lowerCAmelCase    : Union[str, Any]		= ProcessorMixin.push_to_hub.__doc__.format(
      object="processor", object_class="AutoProcessor", object_files="processor files"
  )











import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer
class       _UpperCamelCase   (					tf.keras.layers.Layer		):
      def __init__(      self :int			,   lowerCamelCase :List			,   lowerCamelCase :bool			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :int = None			,   lowerCamelCase :str = "longest"			,   lowerCamelCase :bool = True			,   lowerCamelCase :int = 512			,   lowerCamelCase :int = None			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   lowerCamelCase :bool = True			,   **lowerCamelCase :Optional[Any]			,   )  ->      List[str]:
       super().__init__()
       if use_fast_bert_tokenizer:
        UpperCAmelCase__							=    FastBertTokenizer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case_nfd_strip_accents=lowerCamelCase			,   **lowerCamelCase   )
       else:
        UpperCAmelCase__							=    tf.lookup.StaticVocabularyTable(
            tf.lookup.KeyValueTensorInitializer(
                keys=lowerCamelCase			,   key_dtype=tf.string			,   values=tf.range(tf.size(lowerCamelCase			,   out_type=tf.intaa   )			,   dtype=tf.intaa   )			,   value_dtype=tf.intaa			,   )			,   num_oov_buckets=1			,   )
        UpperCAmelCase__							=    BertTokenizerLayer(
            lowerCamelCase			,   token_out_type=tf.intaa			,   lower_case=lowerCamelCase			,   **lowerCamelCase   )

       UpperCAmelCase__							=    vocab_list
       UpperCAmelCase__							=    do_lower_case
       UpperCAmelCase__							=    cls_token_id or vocab_list.index("[CLS]"   )
       UpperCAmelCase__							=    sep_token_id or vocab_list.index("[SEP]"   )
       UpperCAmelCase__							=    pad_token_id or vocab_list.index("[PAD]"   )
       UpperCAmelCase__							=    ShrinkLongestTrimmer(max_length - 3			,   axis=1   )  # Allow room for special tokens
       UpperCAmelCase__							=    max_length
       UpperCAmelCase__							=    padding
       UpperCAmelCase__							=    truncation
       UpperCAmelCase__							=    pad_to_multiple_of
       UpperCAmelCase__							=    return_token_type_ids
       UpperCAmelCase__							=    return_attention_mask
      @classmethod
      def 						UpperCAmelCase_    (      cls :List[str]			,   lowerCamelCase :"PreTrainedTokenizerBase"			,   **lowerCamelCase :Dict   )  ->      Dict:  # noqa: F821
       UpperCAmelCase__							=    kwargs.pop("do_lower_case"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
       UpperCAmelCase__							=    kwargs.pop("cls_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
       UpperCAmelCase__							=    kwargs.pop("sep_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
       UpperCAmelCase__							=    kwargs.pop("pad_token_id"			,   lowerCamelCase   )
       UpperCAmelCase__							=    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

       UpperCAmelCase__							=    tokenizer.get_vocab()
       UpperCAmelCase__							=    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,   key=lambda lowerCamelCase   : x[1]   )
       UpperCAmelCase__							=    [entry[0] for entry in vocab]
       return cls(
           vocab_list=lowerCamelCase			,   do_lower_case=lowerCamelCase			,   cls_token_id=lowerCamelCase			,   sep_token_id=lowerCamelCase			,   pad_token_id=lowerCamelCase			,   **lowerCamelCase			,   )
      @classmethod
      def 						UpperCAmelCase_    (      cls :Any			,   lowerCamelCase :Union[str, os.PathLike]			,   *lowerCamelCase :str			,   **lowerCamelCase :List[str]   )  ->      Tuple:
       try:
        UpperCAmelCase__							=    BertTokenizer.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       except:  # noqa: E722
        from .tokenization_bert_fast import BertTokenizerFast

        UpperCAmelCase__							=    BertTokenizerFast.from_pretrained(lowerCamelCase			,   *lowerCamelCase			,   **lowerCamelCase   )
       return cls.from_tokenizer(lowerCamelCase			,   **lowerCamelCase   )
      def 						UpperCAmelCase_    (      self :int			,   lowerCamelCase :Optional[int]   )  ->      List[str]:
       if self.do_lower_case:
        UpperCAmelCase__							=    case_fold_utfa(lowerCamelCase   )
       UpperCAmelCase__							=    self.tf_tokenizer.tokenize(lowerCamelCase   )
       return tokens.merge_dims(1			,   -1   )
      def 						UpperCAmelCase_    (      self :Optional[Any]			,   lowerCamelCase :Tuple			,   lowerCamelCase :str=None			,   lowerCamelCase :Tuple=None			,   lowerCamelCase :Any=None			,   lowerCamelCase :Optional[int]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Optional[Any]=None			,   lowerCamelCase :Tuple=None			,   )  ->      str:
       if padding is None:
        UpperCAmelCase__							=    self.padding
       if padding not in ("longest", "max_length"):
        raise ValueError("Padding must be either 'longest' or 'max_length'!"   )
       if max_length is not None and text_pair is not None:
        # Because we have to instantiate a Trimmer to do it properly
        raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"   )
       if max_length is None:
        UpperCAmelCase__							=    self.max_length
       if truncation is None:
        UpperCAmelCase__							=    self.truncation
       if pad_to_multiple_of is None:
        UpperCAmelCase__							=    self.pad_to_multiple_of
       if return_token_type_ids is None:
        UpperCAmelCase__							=    self.return_token_type_ids
       if return_attention_mask is None:
        UpperCAmelCase__							=    self.return_attention_mask
       if not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None and not isinstance(lowerCamelCase			,   tf.Tensor   ):
        UpperCAmelCase__							=    tf.convert_to_tensor(lowerCamelCase   )
       if text_pair is not None:
        if text.shape.rank > 1:
         raise ValueError("text argument should not be multidimensional when a text pair is supplied!"   )
        if text_pair.shape.rank > 1:
         raise ValueError("text_pair should not be multidimensional!"   )
       if text.shape.rank == 2:
        UpperCAmelCase__			,			UpperCAmelCase__							=    text[:, 0], text[:, 1]
       UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
       if text_pair is None:  # Unpaired text
        if truncation:
         UpperCAmelCase__							=    text[:, : max_length - 2]  # Allow room for special tokens
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text,)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       else:  # Paired text
        UpperCAmelCase__							=    self.unpaired_tokenize(lowerCamelCase   )
        if truncation:
         UpperCAmelCase__			,			UpperCAmelCase__							=    self.paired_trimmer.trim([text, text_pair]   )
        UpperCAmelCase__			,			UpperCAmelCase__							=    combine_segments(
            (text, text_pair)			,   start_of_sequence_id=self.cls_token_id			,   end_of_segment_id=self.sep_token_id   )
       if padding == "longest":
        UpperCAmelCase__							=    input_ids.bounding_shape(axis=1   )
        if pad_to_multiple_of is not None:
         # No ceiling division in tensorflow, so we negate floordiv instead
         UpperCAmelCase__							=    pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,   lowerCamelCase   ))
       else:
        UpperCAmelCase__							=    max_length

       UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
       UpperCAmelCase__							=    {"input_ids": input_ids}
       if return_attention_mask:
        UpperCAmelCase__							=    attention_mask
       if return_token_type_ids:
        UpperCAmelCase__			,			UpperCAmelCase__							=    pad_model_inputs(
            lowerCamelCase			,   max_seq_length=lowerCamelCase			,   pad_value=self.pad_token_id   )
        UpperCAmelCase__							=    token_type_ids
       return output







      def 						UpperCAmelCase_    (      self :Tuple   )  ->      Tuple:
       return {
           "vocab_list": self.vocab_list,
           "do_lower_case": self.do_lower_case,
           "cls_token_id": self.cls_token_id,
           "sep_token_id": self.sep_token_id,
           "pad_token_id": self.pad_token_id,
       }





