




from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy


class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase    ):
					_lowerCamelCase						    =					"""MarkupLMFeatureExtractor"""
					_lowerCamelCase						    =					("""MarkupLMTokenizer""", """MarkupLMTokenizerFast""")
					_lowerCamelCase						    =					True

					def __call__(							self					,			__lowercase=None					,			__lowercase=None					,			__lowercase=None					,			__lowercase=None					,			__lowercase=None					,			__lowercase = True					,			__lowercase = False					,			__lowercase = None					,			__lowercase = None					,			__lowercase = 0					,			__lowercase = None					,			__lowercase = None					,			__lowercase = None					,			__lowercase = False					,			__lowercase = False					,			__lowercase = False					,			__lowercase = False					,			__lowercase = True					,			__lowercase = None					,			**__lowercase					,			):







						'''simple docstring'''




						if self.parse_html:
							if html_strings is None:
								raise ValueError('Make sure to pass HTML strings in case `parse_html` is set to `True`'   )

							if nodes is not None or xpaths is not None or node_labels is not None:
								raise ValueError(
								    'Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'   )

							lowerCamelCase_				:						Dict				  =				self.feature_extractor(__lowercase   )
							lowerCamelCase_				:						Any				  =				features['nodes']
							lowerCamelCase_				:						Union[str, Any]				  =				features['xpaths']
						else:
							if html_strings is not None:
								raise ValueError('You have passed HTML strings but `parse_html` is set to `False`.'   )
							if nodes is None or xpaths is None:
								raise ValueError('Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'   )

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
							if isinstance(__lowercase					,			__lowercase   ):
								lowerCamelCase_				:						Optional[Any]				  =				[questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCamelCase_				:						Optional[int]				  =				self.tokenizer(
						    text=questions if questions is not None else nodes					,			text_pair=nodes if questions is not None else None					,			xpaths=__lowercase					,			node_labels=__lowercase					,			add_special_tokens=__lowercase					,			padding=__lowercase					,			truncation=__lowercase					,			max_length=__lowercase					,			stride=__lowercase					,			pad_to_multiple_of=__lowercase					,			return_token_type_ids=__lowercase					,			return_attention_mask=__lowercase					,			return_overflowing_tokens=__lowercase					,			return_special_tokens_mask=__lowercase					,			return_offsets_mapping=__lowercase					,			return_length=__lowercase					,			verbose=__lowercase					,			return_tensors=__lowercase					,			**__lowercase					,			)

						return encoded_inputs

					def       lowercase_				(							self					,			*__lowercase					,			**__lowercase   ):







						'''simple docstring'''




						return self.tokenizer.batch_decode(*__lowercase					,			**__lowercase   )

					def       lowercase_				(							self					,			*__lowercase					,			**__lowercase   ):







						'''simple docstring'''




						return self.tokenizer.decode(*__lowercase					,			**__lowercase   )





					@property
					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[Any]				  =				self.tokenizer.model_input_names
						return tokenizer_input_names


"""simple docstring"""
import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import SegformerImageProcessor






class   _UpperCAmelCase					(       unittest.TestCase):
    def __init__(     self    ,     snake_case_    ,     snake_case_=7    ,     snake_case_=3    ,     snake_case_=30    ,     snake_case_=4_00    ,     snake_case_=True    ,     snake_case_=None    ,     snake_case_=True    ,     snake_case_=[0.5, 0.5, 0.5]    ,     snake_case_=[0.5, 0.5, 0.5]    ,     snake_case_=False    ,     ):
      _snake_case  :    List[str]            = size if size is not None else {"height": 30, "width": 30}
      _snake_case  :    Optional[int]            = parent
      _snake_case  :    Dict            = batch_size
      _snake_case  :    Any            = num_channels
      _snake_case  :    Optional[Any]            = min_resolution
      _snake_case  :    int            = max_resolution
      _snake_case  :    int            = do_resize
      _snake_case  :    List[Any]            = size
      _snake_case  :    int            = do_normalize
      _snake_case  :    Any            = image_mean
      _snake_case  :    str            = image_std
      _snake_case  :    List[Any]            = do_reduce_labels


    def        lowerCamelCase__						(     self  ):
      return {
          "do_resize": self.do_resize,
          "size": self.size,
          "do_normalize": self.do_normalize,
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_reduce_labels": self.do_reduce_labels,
      }


def 					a__ ( ):

  """simple docstring"""







  _snake_case  :    List[str]            = load_dataset("hf-internal-testing/fixtures_ade20k"							,  split="test"		)

  _snake_case  :    Any            = Image.open(dataset[0]["file"]		)
  _snake_case  :    Optional[int]            = Image.open(dataset[1]["file"]		)

  return image, map
def 					a__ ( ):

  """simple docstring"""







  _snake_case  :    Optional[int]            = load_dataset("hf-internal-testing/fixtures_ade20k"							,  split="test"		)

  _snake_case  :    Optional[int]            = Image.open(dataset[0]["file"]		)
  _snake_case  :    List[str]            = Image.open(dataset[1]["file"]		)
  _snake_case  :    Dict            = Image.open(dataset[2]["file"]		)
  _snake_case  :    List[str]            = Image.open(dataset[3]["file"]		)

  return [imagea, imagea], [mapa, mapa]






@require_torch
@require_vision
class   _UpperCAmelCase					(       _snake_case		,       unittest.TestCase):
    __lowercase				:			Dict	=       SegformerImageProcessor if is_vision_available() else None
    def        lowerCamelCase__						(     self  ):
      _snake_case  :    List[str]            = SegformerImageProcessingTester(self  )
    @property
    def        lowerCamelCase__						(     self  ):
      return self.image_processor_tester.prepare_image_processor_dict()
    def        lowerCamelCase__						(     self  ):
      _snake_case  :    List[Any]            = self.image_processing_class(**self.image_processor_dict  )
      self.assertTrue(hasattr(snake_case_    ,     "do_resize"  )  )
      self.assertTrue(hasattr(snake_case_    ,     "size"  )  )
      self.assertTrue(hasattr(snake_case_    ,     "do_normalize"  )  )
      self.assertTrue(hasattr(snake_case_    ,     "image_mean"  )  )
      self.assertTrue(hasattr(snake_case_    ,     "image_std"  )  )
      self.assertTrue(hasattr(snake_case_    ,     "do_reduce_labels"  )  )
    def        lowerCamelCase__						(     self  ):
      _snake_case  :    List[str]            = self.image_processing_class.from_dict(self.image_processor_dict  )
      self.assertEqual(image_processor.size    ,     {"height": 30, "width": 30}  )
      self.assertEqual(image_processor.do_reduce_labels    ,     snake_case_  )

      _snake_case  :    int            = self.image_processing_class.from_dict(self.image_processor_dict    ,     size=42    ,     reduce_labels=snake_case_  )
      self.assertEqual(image_processor.size    ,     {"height": 42, "width": 42}  )
      self.assertEqual(image_processor.do_reduce_labels    ,     snake_case_  )
    def        lowerCamelCase__						(     self  ):
      pass
    def        lowerCamelCase__						(     self  ):
      # Initialize image_processing
      _snake_case  :    List[Any]            = self.image_processing_class(**self.image_processor_dict  )
      # create random PIL images
      _snake_case  :    List[str]            = prepare_image_inputs(self.image_processor_tester    ,     equal_resolution=snake_case_  )
      for image in image_inputs:
        self.assertIsInstance(snake_case_    ,     Image.Image  )

      # Test not batched input
      _snake_case  :    Any            = image_processing(image_inputs[0]    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )

      # Test batched
      _snake_case  :    List[Any]            = image_processing(snake_case_    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
    def        lowerCamelCase__						(     self  ):
      # Initialize image_processing
      _snake_case  :    Optional[Any]            = self.image_processing_class(**self.image_processor_dict  )
      # create random numpy tensors
      _snake_case  :    Any            = prepare_image_inputs(self.image_processor_tester    ,     equal_resolution=snake_case_    ,     numpify=snake_case_  )
      for image in image_inputs:
        self.assertIsInstance(snake_case_    ,     np.ndarray  )

      # Test not batched input
      _snake_case  :    Optional[int]            = image_processing(image_inputs[0]    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )

      # Test batched
      _snake_case  :    Optional[int]            = image_processing(snake_case_    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
    def        lowerCamelCase__						(     self  ):
      # Initialize image_processing
      _snake_case  :    Optional[int]            = self.image_processing_class(**self.image_processor_dict  )
      # create random PyTorch tensors
      _snake_case  :    Optional[Any]            = prepare_image_inputs(self.image_processor_tester    ,     equal_resolution=snake_case_    ,     torchify=snake_case_  )
      for image in image_inputs:
        self.assertIsInstance(snake_case_    ,     torch.Tensor  )

      # Test not batched input
      _snake_case  :    Dict            = image_processing(image_inputs[0]    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )

      # Test batched
      _snake_case  :    List[Any]            = image_processing(snake_case_    ,     return_tensors="pt"  ).pixel_values
      self.assertEqual(
          encoded_images.shape    ,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
    def        lowerCamelCase__						(     self  ):
      # Initialize image_processing
      _snake_case  :    int            = self.image_processing_class(**self.image_processor_dict  )
      # create random PyTorch tensors
      _snake_case  :    List[str]            = prepare_image_inputs(self.image_processor_tester    ,     equal_resolution=snake_case_    ,     torchify=snake_case_  )
      _snake_case  :    Optional[int]            = []
      for image in image_inputs:
        self.assertIsInstance(snake_case_    ,     torch.Tensor  )
        maps.append(torch.zeros(image.shape[-2:]  ).long()  )

      # Test not batched input
      _snake_case  :    Optional[Any]            = image_processing(image_inputs[0]    ,     maps[0]    ,     return_tensors="pt"  )
      self.assertEqual(
          encoding["pixel_values"].shape    ,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(
          encoding["labels"].shape    ,     (
              1,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(encoding["labels"].dtype    ,     torch.long  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 2_55  )

      # Test batched
      _snake_case  :    List[Any]            = image_processing(snake_case_    ,     snake_case_    ,     return_tensors="pt"  )
      self.assertEqual(
          encoding["pixel_values"].shape    ,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(
          encoding["labels"].shape    ,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(encoding["labels"].dtype    ,     torch.long  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 2_55  )

      # Test not batched input (PIL images)
      _snake_case      ,       _snake_case  :    int            = prepare_semantic_single_inputs()

      _snake_case  :    List[str]            = image_processing(snake_case_    ,     snake_case_    ,     return_tensors="pt"  )
      self.assertEqual(
          encoding["pixel_values"].shape    ,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(
          encoding["labels"].shape    ,     (
              1,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(encoding["labels"].dtype    ,     torch.long  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 2_55  )

      # Test batched input (PIL images)
      _snake_case      ,       _snake_case  :    str            = prepare_semantic_batch_inputs()

      _snake_case  :    List[Any]            = image_processing(snake_case_    ,     snake_case_    ,     return_tensors="pt"  )
      self.assertEqual(
          encoding["pixel_values"].shape    ,     (
              2,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(
          encoding["labels"].shape    ,     (
              2,
              self.image_processor_tester.size["height"],
              self.image_processor_tester.size["width"],
          )    ,     )
      self.assertEqual(encoding["labels"].dtype    ,     torch.long  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 2_55  )


    def        lowerCamelCase__						(     self  ):
      # Initialize image_processing
      _snake_case  :    List[Any]            = self.image_processing_class(**self.image_processor_dict  )

      # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
      _snake_case      ,       _snake_case  :    List[Any]            = prepare_semantic_single_inputs()
      _snake_case  :    Dict            = image_processing(snake_case_    ,     snake_case_    ,     return_tensors="pt"  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 1_50  )

      _snake_case  :    Tuple            = True
      _snake_case  :    Union[str, Any]            = image_processing(snake_case_    ,     snake_case_    ,     return_tensors="pt"  )
      self.assertTrue(encoding["labels"].min().item() >= 0  )
      self.assertTrue(encoding["labels"].max().item() <= 2_55  )

