
'''simple docstring'''







import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


lowerCAmelCase_      =   logging.get_logger(__name__)

lowerCAmelCase_      =   OrderedDict(
    [
        ("align", "AlignProcessor"),
        ("altclip", "AltCLIPProcessor"),
        ("bark", "BarkProcessor"),
        ("blip", "BlipProcessor"),
        ("blip-2", "Blip2Processor"),
        ("bridgetower", "BridgeTowerProcessor"),
        ("chinese_clip", "ChineseCLIPProcessor"),
        ("clap", "ClapProcessor"),
        ("clip", "CLIPProcessor"),
        ("clipseg", "CLIPSegProcessor"),
        ("flava", "FlavaProcessor"),
        ("git", "GitProcessor"),
        ("groupvit", "CLIPProcessor"),
        ("hubert", "Wav2Vec2Processor"),
        ("instructblip", "InstructBlipProcessor"),
        ("layoutlmv2", "LayoutLMv2Processor"),
        ("layoutlmv3", "LayoutLMv3Processor"),
        ("markuplm", "MarkupLMProcessor"),
        ("mctct", "MCTCTProcessor"),
        ("mgp-str", "MgpstrProcessor"),
        ("oneformer", "OneFormerProcessor"),
        ("owlvit", "OwlViTProcessor"),
        ("pix2struct", "Pix2StructProcessor"),
        ("sam", "SamProcessor"),
        ("sew", "Wav2Vec2Processor"),
        ("sew-d", "Wav2Vec2Processor"),
        ("speech_to_text", "Speech2TextProcessor"),
        ("speech_to_text_2", "Speech2Text2Processor"),
        ("speecht5", "SpeechT5Processor"),
        ("trocr", "TrOCRProcessor"),
        ("tvlt", "TvltProcessor"),
        ("unispeech", "Wav2Vec2Processor"),
        ("unispeech-sat", "Wav2Vec2Processor"),
        ("vilt", "ViltProcessor"),
        ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"),
        ("wav2vec2", "Wav2Vec2Processor"),
        ("wav2vec2-conformer", "Wav2Vec2Processor"),
        ("wavlm", "Wav2Vec2Processor"),
        ("whisper", "WhisperProcessor"),
        ("xclip", "XCLIPProcessor"),
    ]
)

lowerCAmelCase_      =   _LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)







def   lowerCAmelCase(  a__    :  str      ):







       '''simple docstring'''






       for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
              if class_name in processors:
                     lowerCamelCase__				   =      model_type_to_module_name(a__      )

                     lowerCamelCase__				   =      importlib.import_module(f""".{module_name}"""      , "transformers.models"      )
                     try:
                            return getattr(a__      , a__      )
                     except AttributeError:
                            continue

       for processor in PROCESSOR_MAPPING._extra_content.values():
              if getattr(a__      , "__name__"      , a__      ) == class_name:
                     return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
       lowerCamelCase__				   =      importlib.import_module("transformers"      )
       if hasattr(a__      , a__      ):
              return getattr(a__      , a__      )

       return None







class 		snake_case_       :



   """simple docstring"""





   def __init__( self):
          raise EnvironmentError(
              "AutoProcessor is designed to be instantiated "
              "using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method.")







   @classmethod
   @replace_list_option_in_docstrings(UpperCamelCase)
   def     __UpperCAmelCase      ( cls			,   UpperCamelCase			,   **UpperCamelCase):
          lowerCamelCase__				   =      kwargs.pop("config"			,   UpperCamelCase)
          lowerCamelCase__				   =      kwargs.pop("trust_remote_code"			,   UpperCamelCase)
          lowerCamelCase__				   =      True

          lowerCamelCase__				   =      None
          lowerCamelCase__				   =      None

          # First, let's see if we have a preprocessor config.
          # Filter the kwargs for `get_file_from_repo`.
          lowerCamelCase__				   =      {
              key: kwargs[key] for key in inspect.signature(UpperCamelCase).parameters.keys() if key in kwargs
          }
          # Let's start by checking whether the processor class is saved in an image processor
          lowerCamelCase__				   =      get_file_from_repo(
              UpperCamelCase			,   UpperCamelCase			,   **UpperCamelCase)
          if preprocessor_config_file is not None:
                 lowerCamelCase__   ,		lowerCamelCase__				   =      ImageProcessingMixin.get_image_processor_dict(UpperCamelCase			,   **UpperCamelCase)
                 lowerCamelCase__				   =      config_dict.get("processor_class"			,   UpperCamelCase)
                 if "AutoProcessor" in config_dict.get("auto_map"			,   {}):
                        lowerCamelCase__				   =      config_dict["auto_map"]["AutoProcessor"]

        # If not found, let's check whether the processor class is saved in a feature extractor config
          if preprocessor_config_file is not None and processor_class is None:
                 lowerCamelCase__   ,		lowerCamelCase__				   =      FeatureExtractionMixin.get_feature_extractor_dict(UpperCamelCase			,   **UpperCamelCase)
                 lowerCamelCase__				   =      config_dict.get("processor_class"			,   UpperCamelCase)
                 if "AutoProcessor" in config_dict.get("auto_map"			,   {}):
                        lowerCamelCase__				   =      config_dict["auto_map"]["AutoProcessor"]

          if processor_class is None:
                 # Next, let's check whether the processor class is saved in a tokenizer
                 lowerCamelCase__				   =      get_file_from_repo(
                     UpperCamelCase			,   UpperCamelCase			,   **UpperCamelCase)
                 if tokenizer_config_file is not None:
                        with open(UpperCamelCase			,   encoding="utf-8") as reader:
                               lowerCamelCase__				   =      json.load(UpperCamelCase)

                        lowerCamelCase__				   =      config_dict.get("processor_class"			,   UpperCamelCase)
                        if "AutoProcessor" in config_dict.get("auto_map"			,   {}):
                               lowerCamelCase__				   =      config_dict["auto_map"]["AutoProcessor"]

          if processor_class is None:
                 # Otherwise, load config, if it can be loaded.
                 if not isinstance(UpperCamelCase			,   UpperCamelCase):
                        lowerCamelCase__				   =      AutoConfig.from_pretrained(
                            UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)

                 # And check if the config contains the processor class.
                 lowerCamelCase__				   =      getattr(UpperCamelCase			,   "processor_class"			,   UpperCamelCase)
                 if hasattr(UpperCamelCase			,   "auto_map") and "AutoProcessor" in config.auto_map:
                        lowerCamelCase__				   =      config.auto_map["AutoProcessor"]

          if processor_class is not None:
                 lowerCamelCase__				   =      processor_class_from_name(UpperCamelCase)

          lowerCamelCase__				   =      processor_auto_map is not None
          lowerCamelCase__				   =      processor_class is not None or type(UpperCamelCase) in PROCESSOR_MAPPING
          lowerCamelCase__				   =      resolve_trust_remote_code(
              UpperCamelCase			,   UpperCamelCase			,   UpperCamelCase			,   UpperCamelCase)

          if has_remote_code and trust_remote_code:
                 lowerCamelCase__				   =      get_class_from_dynamic_module(
                     UpperCamelCase			,   UpperCamelCase			,   **UpperCamelCase)
                 lowerCamelCase__				   =      kwargs.pop("code_revision"			,   UpperCamelCase)
                 if os.path.isdir(UpperCamelCase):
                        processor_class.register_for_auto_class()
                 return processor_class.from_pretrained(
                     UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)
          elif processor_class is not None:
                 return processor_class.from_pretrained(
                     UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)
          # Last try: we use the PROCESSOR_MAPPING.
          elif type(UpperCamelCase) in PROCESSOR_MAPPING:
                 return PROCESSOR_MAPPING[type(UpperCamelCase)].from_pretrained(UpperCamelCase			,   **UpperCamelCase)

          # At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
          # tokenizer.
          try:
                 return AutoTokenizer.from_pretrained(
                     UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)
          except Exception:
                 try:
                        return AutoImageProcessor.from_pretrained(
                            UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)
                 except Exception:
                        pass

                 try:
                        return AutoFeatureExtractor.from_pretrained(
                            UpperCamelCase			,   trust_remote_code=UpperCamelCase			,   **UpperCamelCase)
                 except Exception:
                        pass

          raise ValueError(
              f"""Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a """
              "tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains"
              "the files of at least one of those processing classes.")







   @staticmethod
   def     __UpperCAmelCase      ( UpperCamelCase			,   UpperCamelCase):
          PROCESSOR_MAPPING.register(UpperCamelCase			,   UpperCamelCase)



'''simple docstring'''







import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		snake_case_       (						A__		,	unittest.TestCase						):



   """simple docstring"""





   __lowerCAmelCase	:							Optional[Any]								=HerbertTokenizer
   __lowerCAmelCase	:							Optional[Any]								=HerbertTokenizerFast
   __lowerCAmelCase	:							int								=True







   def     __UpperCAmelCase      ( self):
          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(f"""{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"""			,   encoding="utf-8") as f_data:
                 lowerCamelCase__				   =      f_data.read().replace("\n\n"			,   "\n").strip()

          lowerCamelCase__				   =      [
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          lowerCamelCase__				   =      dict(zip(UpperCamelCase			,   range(len(UpperCamelCase))))
          lowerCamelCase__				   =      ["l o 123", "lo w 1456", "e r</w> 1789", ""]

          lowerCamelCase__				   =      os.path.join(self.tmpdirname			,   VOCAB_FILES_NAMES["vocab_file"])
          lowerCamelCase__				   =      os.path.join(self.tmpdirname			,   VOCAB_FILES_NAMES["merges_file"])
          with open(self.vocab_file			,   "w") as fp:
                 fp.write(json.dumps(UpperCamelCase))
          with open(self.merges_file			,   "w") as fp:
                 fp.write("\n".join(UpperCamelCase))







   def     __UpperCAmelCase      ( self			,   UpperCamelCase):
          lowerCamelCase__				   =      "lower newer"
          lowerCamelCase__				   =      "lower newer"
          return input_text, output_text







   def     __UpperCAmelCase      ( self):
          lowerCamelCase__				   =      self.tokenizer_class(vocab_file=self.vocab_file			,   merges_file=self.merges_file)

          lowerCamelCase__				   =      "lower"
          lowerCamelCase__				   =      ["low", "er</w>"]
          lowerCamelCase__				   =      tokenizer.tokenize(UpperCamelCase)
          self.assertListEqual(UpperCamelCase			,   UpperCamelCase)

          lowerCamelCase__				   =      tokens + ["<unk>"]
          lowerCamelCase__				   =      [16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase)			,   UpperCamelCase)







   def     __UpperCAmelCase      ( self):
          if not self.test_rust_tokenizer:
                 return

          lowerCamelCase__				   =      self.get_tokenizer()
          lowerCamelCase__				   =      self.get_rust_tokenizer()

          lowerCamelCase__				   =      "lower,newer"

          lowerCamelCase__				   =      tokenizer.tokenize(UpperCamelCase)
          lowerCamelCase__				   =      rust_tokenizer.tokenize(UpperCamelCase)
          self.assertListEqual(UpperCamelCase			,   UpperCamelCase)

          lowerCamelCase__				   =      tokenizer.encode(UpperCamelCase			,   add_special_tokens=UpperCamelCase)
          lowerCamelCase__				   =      rust_tokenizer.encode(UpperCamelCase			,   add_special_tokens=UpperCamelCase)
          self.assertListEqual(UpperCamelCase			,   UpperCamelCase)

          lowerCamelCase__				   =      self.get_rust_tokenizer()
          lowerCamelCase__				   =      tokenizer.encode(UpperCamelCase)
          lowerCamelCase__				   =      rust_tokenizer.encode(UpperCamelCase)
          self.assertListEqual(UpperCamelCase			,   UpperCamelCase)







   @slow
   def     __UpperCAmelCase      ( self):
          lowerCamelCase__				   =      self.tokenizer_class.from_pretrained("allegro/herbert-base-cased")

          lowerCamelCase__				   =      tokenizer.encode("konstruowanie sekwencji"			,   add_special_tokens=UpperCamelCase)
          lowerCamelCase__				   =      tokenizer.encode("konstruowanie wielu sekwencji"			,   add_special_tokens=UpperCamelCase)

          lowerCamelCase__				   =      tokenizer.build_inputs_with_special_tokens(UpperCamelCase)
          lowerCamelCase__				   =      tokenizer.build_inputs_with_special_tokens(UpperCamelCase			,   UpperCamelCase)

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]

