



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast

class     _A       (	unittest.TestCase    ):



       def 		_lowerCamelCase			(		self							:     Optional[Any]):


           '''simple docstring'''

           __a						       =				mock.Mock()
           __a						       =				500
           __a						       =				{}
           __a						       =				HTTPError
           __a						       =				{}

           # Download this model to make sure it's in the cache.
           __a						       =				BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch('''requests.Session.request'''     ,		return_value=__SCREAMING_SNAKE_CASE) as mock_head:
               __a						       =				BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
               # This check we did call the fake head request
               mock_head.assert_called()



       @require_tokenizers
       def 		_lowerCamelCase			(		self							:     int):


           '''simple docstring'''

           __a						       =				mock.Mock()
           __a						       =				500
           __a						       =				{}
           __a						       =				HTTPError
           __a						       =				{}

           # Download this model to make sure it's in the cache.
           __a						       =				GPTaTokenizerFast.from_pretrained('''gpt2''')

           # Under the mock environment we get a 500 error when trying to reach the tokenizer.
           with mock.patch('''requests.Session.request'''     ,		return_value=__SCREAMING_SNAKE_CASE) as mock_head:
               __a						       =				GPTaTokenizerFast.from_pretrained('''gpt2''')
               # This check we did call the fake head request
               mock_head.assert_called()



       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           try:
               __a						       =				tempfile.mktemp()
               with open(__SCREAMING_SNAKE_CASE     ,		'''wb''') as f:
                   http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''     ,		__SCREAMING_SNAKE_CASE)

               __a						       =				AlbertTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
           finally:
               os.remove(__SCREAMING_SNAKE_CASE)

           # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
           # the current folder and have the right name.
           if os.path.isfile('''tokenizer.json'''):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
           try:
               with open('''tokenizer.json'''     ,		'''wb''') as f:
                   http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''     ,		__SCREAMING_SNAKE_CASE)
               __a						       =				AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''')
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size     ,		1_000)
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

           finally:
               os.remove('''tokenizer.json''')





       def 		_lowerCamelCase			(		self							:     str):


           '''simple docstring'''

           __a						       =				AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''')


@is_staging_test
class     _A       (	unittest.TestCase    ):
       UpperCamelCase__  :    Dict											=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']



       @classmethod
       def 		_lowerCamelCase			(		cls							:     List[Any]):


           '''simple docstring'''

           __a						       =				TOKEN
           HfFolder.save_token(__SCREAMING_SNAKE_CASE)



       @classmethod
       def 		_lowerCamelCase			(		cls							:     Optional[Any]):


           '''simple docstring'''

           try:
               delete_repo(token=cls._token     ,		repo_id='''test-tokenizer''')
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token     ,		repo_id='''valid_org/test-tokenizer-org''')
           except HTTPError:
               pass

           try:
               delete_repo(token=cls._token     ,		repo_id='''test-dynamic-tokenizer''')
           except HTTPError:
               pass



       def 		_lowerCamelCase			(		self							:     List[str]):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               __a						       =				os.path.join(__SCREAMING_SNAKE_CASE     ,		'''vocab.txt''')
               with open(__SCREAMING_SNAKE_CASE     ,		'''w'''     ,		encoding='''utf-8''') as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
               __a						       =				BertTokenizer(__SCREAMING_SNAKE_CASE)

           tokenizer.push_to_hub('''test-tokenizer'''     ,		use_auth_token=self._token)
           __a						       =				BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
           self.assertDictEqual(new_tokenizer.vocab     ,		tokenizer.vocab)

           # Reset repo
           delete_repo(token=self._token     ,		repo_id='''test-tokenizer''')

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE     ,		repo_id='''test-tokenizer'''     ,		push_to_hub=__SCREAMING_SNAKE_CASE     ,		use_auth_token=self._token)

           __a						       =				BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
           self.assertDictEqual(new_tokenizer.vocab     ,		tokenizer.vocab)



       def 		_lowerCamelCase			(		self							:     Any):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               __a						       =				os.path.join(__SCREAMING_SNAKE_CASE     ,		'''vocab.txt''')
               with open(__SCREAMING_SNAKE_CASE     ,		'''w'''     ,		encoding='''utf-8''') as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
               __a						       =				BertTokenizer(__SCREAMING_SNAKE_CASE)

           tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''     ,		use_auth_token=self._token)
           __a						       =				BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
           self.assertDictEqual(new_tokenizer.vocab     ,		tokenizer.vocab)

           # Reset repo
           delete_repo(token=self._token     ,		repo_id='''valid_org/test-tokenizer-org''')

           # Push to hub via save_pretrained
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   __SCREAMING_SNAKE_CASE     ,		repo_id='''valid_org/test-tokenizer-org'''     ,		push_to_hub=__SCREAMING_SNAKE_CASE     ,		use_auth_token=self._token)

           __a						       =				BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
           self.assertDictEqual(new_tokenizer.vocab     ,		tokenizer.vocab)





       @require_tokenizers
       def 		_lowerCamelCase			(		self							:     Any):


           '''simple docstring'''

           CustomTokenizer.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               __a						       =				os.path.join(__SCREAMING_SNAKE_CASE     ,		'''vocab.txt''')
               with open(__SCREAMING_SNAKE_CASE     ,		'''w'''     ,		encoding='''utf-8''') as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
               __a						       =				CustomTokenizer(__SCREAMING_SNAKE_CASE)

           # No fast custom tokenizer
           tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,		use_auth_token=self._token)

           __a						       =				AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'     ,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,		'''CustomTokenizer''')

           # Fast and slow custom tokenizer
           CustomTokenizerFast.register_for_auto_class()
           with tempfile.TemporaryDirectory() as tmp_dir:
               __a						       =				os.path.join(__SCREAMING_SNAKE_CASE     ,		'''vocab.txt''')
               with open(__SCREAMING_SNAKE_CASE     ,		'''w'''     ,		encoding='''utf-8''') as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))

               __a						       =				BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE)
               bert_tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
               __a						       =				CustomTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE)

           tokenizer.push_to_hub('''test-dynamic-tokenizer'''     ,		use_auth_token=self._token)

           __a						       =				AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'     ,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,		'''CustomTokenizerFast''')
           __a						       =				AutoTokenizer.from_pretrained(
               F'{USER}/test-dynamic-tokenizer'     ,		use_fast=__SCREAMING_SNAKE_CASE     ,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
           self.assertEqual(tokenizer.__class__.__name__     ,		'''CustomTokenizer''')


class     _A       (	unittest.TestCase    ):



       def 		_lowerCamelCase			(		self							:     List[str]):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''Hello 友達''')
           self.assertEqual(trie.data     ,		{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})
           trie.add('''Hello''')
           trie.data
           self.assertEqual(trie.data     ,		{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})



       def 		_lowerCamelCase			(		self							:     Any):


           '''simple docstring'''

           __a						       =				Trie()
           self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')     ,		['''[CLS] This is a extra_id_100'''])
           trie.add('''[CLS]''')
           trie.add('''extra_id_1''')
           trie.add('''extra_id_100''')
           self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')     ,		['''[CLS]''', ''' This is a ''', '''extra_id_100'''])



       def 		_lowerCamelCase			(		self							:     int):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''A''')
           self.assertEqual(trie.split('''ABC''')     ,		['''A''', '''BC'''])
           self.assertEqual(trie.split('''BCA''')     ,		['''BC''', '''A'''])



       def 		_lowerCamelCase			(		self							:     int):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''TOKEN]''')
           trie.add('''[SPECIAL_TOKEN]''')
           self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')     ,		['''This is something ''', '''[SPECIAL_TOKEN]'''])



       def 		_lowerCamelCase			(		self							:     Optional[int]):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''A''')
           trie.add('''P''')
           trie.add('''[SPECIAL_TOKEN]''')
           self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')     ,		['''This is something ''', '''[SPECIAL_TOKEN]'''])



       def 		_lowerCamelCase			(		self							:     Union[str, Any]):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''AB''')
           trie.add('''B''')
           trie.add('''C''')
           self.assertEqual(trie.split('''ABC''')     ,		['''AB''', '''C'''])



       def 		_lowerCamelCase			(		self							:     Any):


           '''simple docstring'''

           __a						       =				Trie()
           trie.add('''ABC''')
           trie.add('''B''')
           trie.add('''CD''')
           self.assertEqual(trie.split('''ABCD''')     ,		['''ABC''', '''D'''])





       def 		_lowerCamelCase			(		self							:     Any):


           '''simple docstring'''

           __a						       =				Trie()
           __a						       =				trie.cut_text('''ABC'''     ,		[0, 0, 2, 1, 2, 3])
           self.assertEqual(__SCREAMING_SNAKE_CASE     ,		['''AB''', '''C'''])










from collections import defaultdict
from typing import Optional

from ..image_utils import load_image
from ..utils import (
    add_end_docstrings,
    is_torch_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ChunkPipeline


if is_torch_available():
 import torch

 from ..models.auto.modeling_auto import MODEL_FOR_MASK_GENERATION_MAPPING

__snake_case						:List[Any]      		=     logging.get_logger(__name__)

@add_end_docstrings(__UpperCAmelCase    )
class     _A       (	__UpperCAmelCase    ):



       def __init__(		self							:     Dict     ,		**__SCREAMING_SNAKE_CASE							:     Tuple):


           '''simple docstring'''

           super().__init__(**__SCREAMING_SNAKE_CASE)
           requires_backends(self     ,		'''vision''')
           requires_backends(self     ,		'''torch''')

           if self.framework != "pt":
               raise ValueError(F'The {self.__class__} is only available in PyTorch.')

           self.check_model_type(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Any     ,		**__SCREAMING_SNAKE_CASE							:     Any):


           '''simple docstring'''

           __a						       =				{}
           __a						       =				{}
           __a						       =				{}
           # preprocess args
           if "points_per_batch" in kwargs:
               __a						       =				kwargs['''points_per_batch''']
           if "points_per_crop" in kwargs:
               __a						       =				kwargs['''points_per_crop''']
           if "crops_n_layers" in kwargs:
               __a						       =				kwargs['''crops_n_layers''']
           if "crop_overlap_ratio" in kwargs:
               __a						       =				kwargs['''crop_overlap_ratio''']
           if "crop_n_points_downscale_factor" in kwargs:
               __a						       =				kwargs['''crop_n_points_downscale_factor''']
           # postprocess args
           if "pred_iou_thresh" in kwargs:
               __a						       =				kwargs['''pred_iou_thresh''']
           if "stability_score_offset" in kwargs:
               __a						       =				kwargs['''stability_score_offset''']
           if "mask_threshold" in kwargs:
               __a						       =				kwargs['''mask_threshold''']
           if "stability_score_thresh" in kwargs:
               __a						       =				kwargs['''stability_score_thresh''']
           if "crops_nms_thresh" in kwargs:
               __a						       =				kwargs['''crops_nms_thresh''']
           if "output_rle_mask" in kwargs:
               __a						       =				kwargs['''output_rle_mask''']
           if "output_bboxes_mask" in kwargs:
               __a						       =				kwargs['''output_bboxes_mask''']
           return preprocess_kwargs, forward_params, postprocess_kwargs



       def __call__(		self							:     Union[str, Any]     ,		__SCREAMING_SNAKE_CASE							:     int     ,		*__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     Tuple=None     ,		__SCREAMING_SNAKE_CASE							:     Any=None     ,		**__SCREAMING_SNAKE_CASE							:     str):


           '''simple docstring'''

           return super().__call__(__SCREAMING_SNAKE_CASE     ,		*__SCREAMING_SNAKE_CASE     ,		num_workers=__SCREAMING_SNAKE_CASE     ,		batch_size=__SCREAMING_SNAKE_CASE     ,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase			(		self							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     str     ,		__SCREAMING_SNAKE_CASE							:     Any=64     ,		__SCREAMING_SNAKE_CASE							:     int = 0     ,		__SCREAMING_SNAKE_CASE							:     float = 512 / 1_500     ,		__SCREAMING_SNAKE_CASE							:     Optional[int] = 32     ,		__SCREAMING_SNAKE_CASE							:     Optional[int] = 1     ,		):


           '''simple docstring'''

           __a						       =				load_image(__SCREAMING_SNAKE_CASE)
           __a						       =				self.image_processor.size['''longest_edge''']
           __a       ,   __a       ,   __a       ,   __a						       =				self.image_processor.generate_crop_boxes(
               __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)
           __a						       =				self.image_processor(images=__SCREAMING_SNAKE_CASE     ,		return_tensors='''pt''')

           with self.device_placement():
               if self.framework == "pt":
                   __a						       =				self.get_inference_context()
                   with inference_context():
                       __a						       =				self._ensure_tensor_on_device(__SCREAMING_SNAKE_CASE     ,		device=self.device)
                       __a						       =				self.model.get_image_embeddings(model_inputs.pop('''pixel_values'''))
                       __a						       =				image_embeddings

           __a						       =				grid_points.shape[1]
           __a						       =				points_per_batch if points_per_batch is not None else n_points

           if points_per_batch <= 0:
               raise ValueError(
                   '''Cannot have points_per_batch<=0. Must be >=1 to returned batched outputs. '''
                   '''To return all points at once, set points_per_batch to None''')

           for i in range(0     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE):
               __a						       =				grid_points[:, i : i + points_per_batch, :, :]
               __a						       =				input_labels[:, i : i + points_per_batch]
               __a						       =				i == n_points - points_per_batch
               yield {
                   "input_points": batched_points,
                   "input_labels": labels,
                   "input_boxes": crop_boxes,
                   "is_last": is_last,
                   **model_inputs,
               }



       def 		_lowerCamelCase			(		self							:     Any     ,		__SCREAMING_SNAKE_CASE							:     Optional[Any]     ,		__SCREAMING_SNAKE_CASE							:     int=0.88     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=0.95     ,		__SCREAMING_SNAKE_CASE							:     List[Any]=0     ,		__SCREAMING_SNAKE_CASE							:     int=1     ,		):


           '''simple docstring'''

           __a						       =				model_inputs.pop('''input_boxes''')
           __a						       =				model_inputs.pop('''is_last''')
           __a						       =				model_inputs.pop('''original_sizes''').tolist()
           __a						       =				model_inputs.pop('''reshaped_input_sizes''').tolist()

           __a						       =				self.model(**__SCREAMING_SNAKE_CASE)

           # post processing happens here in order to avoid CPU GPU copies of ALL the masks
           __a						       =				model_outputs['''pred_masks''']
           __a						       =				self.image_processor.post_process_masks(
               __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		binarize=__SCREAMING_SNAKE_CASE)
           __a						       =				model_outputs['''iou_scores''']
           __a       ,   __a       ,   __a						       =				self.image_processor.filter_masks(
               masks[0]     ,		iou_scores[0]     ,		original_sizes[0]     ,		input_boxes[0]     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		)
           return {
               "masks": masks,
               "is_last": is_last,
               "boxes": boxes,
               "iou_scores": iou_scores,
           }





       def 		_lowerCamelCase			(		self							:     int     ,		__SCREAMING_SNAKE_CASE							:     Dict     ,		__SCREAMING_SNAKE_CASE							:     Dict=False     ,		__SCREAMING_SNAKE_CASE							:     Tuple=False     ,		__SCREAMING_SNAKE_CASE							:     int=0.7     ,		):


           '''simple docstring'''

           __a						       =				[]
           __a						       =				[]
           __a						       =				[]
           for model_output in model_outputs:
               all_scores.append(model_output.pop('''iou_scores'''))
               all_masks.extend(model_output.pop('''masks'''))
               all_boxes.append(model_output.pop('''boxes'''))

           __a						       =				torch.cat(__SCREAMING_SNAKE_CASE)
           __a						       =				torch.cat(__SCREAMING_SNAKE_CASE)
           __a       ,   __a       ,   __a       ,   __a						       =				self.image_processor.post_process_for_mask_generation(
               __SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE     ,		__SCREAMING_SNAKE_CASE)

           __a						       =				defaultdict(__SCREAMING_SNAKE_CASE)
           for output in model_outputs:
               for k, v in output.items():
                   extra[k].append(__SCREAMING_SNAKE_CASE)

           __a						       =				{}
           if output_rle_mask:
               __a						       =				rle_mask

           if output_bboxes_mask:
               __a						       =				bounding_boxes

           return {"masks": output_masks, "scores": iou_scores, **optional, **extra}





