




"""simple docstring"""





from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


_lowercase						   =     logging.get_logger(__name__)

_lowercase						   =     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase						   =     {
    """vocab_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """openai/clip-vit-base-patch32""": """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """openai/clip-vit-base-patch32""": (
            """https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase						   =     {
    """openai/clip-vit-base-patch32""": 77,
}

class    __lowercase							(      _UpperCAmelCase						):


				'''simple docstring'''


				_lowerCAmelCase						      =VOCAB_FILES_NAMES
				_lowerCAmelCase						      =PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase						      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase						      =["input_ids", "attention_mask"]
				_lowerCAmelCase						      =CLIPTokenizer

				def __init__( self :  Optional[Any]   ,SCREAMING_SNAKE_CASE_ :  List[str]=None   ,SCREAMING_SNAKE_CASE_ :  Optional[int]=None   ,SCREAMING_SNAKE_CASE_ :  Optional[int]=None   ,SCREAMING_SNAKE_CASE_ :  Any="<|endoftext|>"   ,SCREAMING_SNAKE_CASE_ :  List[str]="<|startoftext|>"   ,SCREAMING_SNAKE_CASE_ :  Optional[int]="<|endoftext|>"   ,SCREAMING_SNAKE_CASE_ :  List[Any]="<|endoftext|>"   ,**SCREAMING_SNAKE_CASE_ :  Tuple   ,):
										super().__init__(
										    lowercase_   ,lowercase_   ,tokenizer_file=lowercase_   ,unk_token=lowercase_   ,bos_token=lowercase_   ,eos_token=lowercase_   ,pad_token=lowercase_   ,**lowercase_   ,)

										if not isinstance(self.backend_tokenizer.pre_tokenizer   ,pre_tokenizers.Sequence   ):
																raise ValueError(
																    '''The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'''
																    ''' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'''
																    ''' to be compatible with this version.The easiest way to do so is'''
																    ''' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'''
																    ''' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'''
																    ''' transformers.'''   )

										self._wrap_decode_method_backend_tokenizer()

				def    _lowerCamelCase       ( self :  Optional[Any]   ):
										snake_case__							:       Optional[int]				      =       self.backend_tokenizer.decode

										def new_decode_method(*SCREAMING_SNAKE_CASE_ :  int   ,**SCREAMING_SNAKE_CASE_ :  int   ):
																snake_case__							:       str				      =       orig_decode_method(*lowercase_   ,**lowercase_   )
																snake_case__							:       int				      =       text.replace(self.backend_tokenizer.model.end_of_word_suffix   ,''' '''   ).strip()
																return text

										snake_case__							:       Optional[int]				      =       new_decode_method

				def    _lowerCamelCase       ( self :  str   ,SCREAMING_SNAKE_CASE_ :  List[int]   ,SCREAMING_SNAKE_CASE_ :  Optional[List[int]] = None   ):
										snake_case__							:       Optional[Any]				      =       [self.bos_token_id]
										snake_case__							:       Tuple				      =       [self.eos_token_id]

										if token_ids_a is None:
																return bos_token + token_ids_a + eos_token
										return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token

				def    _lowerCamelCase       ( self :  Tuple   ,SCREAMING_SNAKE_CASE_ :  List[int]   ,SCREAMING_SNAKE_CASE_ :  Optional[List[int]] = None   ):
										snake_case__							:       Any				      =       [self.bos_token_id]
										snake_case__							:       Union[str, Any]				      =       [self.eos_token_id]

										if token_ids_a is None:
																return len(bos_token + token_ids_a + eos_token   ) * [0]
										return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token   ) * [0]





				def    _lowerCamelCase       ( self :  Tuple   ,SCREAMING_SNAKE_CASE_ :  str   ,SCREAMING_SNAKE_CASE_ :  Optional[str] = None   ):
										snake_case__							:       List[Any]				      =       self._tokenizer.model.save(lowercase_   ,name=lowercase_   )
										return tuple(lowercase_   )





import itertools
import random
import unittest

import numpy as np

from transformers import EncodecFeatureExtractor
from transformers.testing_utils import require_torch
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
     import torch


lowercase_									=    random.Random()
def 		_snake_case(       SCREAMING_SNAKE_CASE__     :      Optional[int]       ,     SCREAMING_SNAKE_CASE__     :      List[str]=1.0       ,     SCREAMING_SNAKE_CASE__     :      List[Any]=None       ,     SCREAMING_SNAKE_CASE__     :      List[Any]=None )				-> List[str]:





    '''simple docstring'''






    if rng is None:
        A__        =  global_rng

    A__        =  []
    for batch_idx in range(shape[0] ):
        values.append([] )
        for _ in range(shape[1] ):
            values[-1].append(rng.random() * scale )

    return values


@require_torch
class 					A		(				unittest.TestCase					):







    """simple docstring"""





    def __init__( self					:			Any,lowercase_					:			Any,lowercase_					:			Tuple=7,lowercase_					:			Union[str, Any]=4_0_0,lowercase_					:			Union[str, Any]=2_0_0_0,lowercase_					:			int=1,lowercase_					:			Optional[Any]=0.0,lowercase_					:			Optional[int]=2_4_0_0_0,lowercase_					:			List[str]=True,)->       Any:





        '''simple docstring'''







        A__        =  parent
        A__        =  batch_size
        A__        =  min_seq_length
        A__        =  max_seq_length
        A__        =  (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
        A__        =  feature_size
        A__        =  padding_value
        A__        =  sampling_rate
        A__        =  return_attention_mask


    def 		snake_case__	( self					:			str						)->       List[str]:





        '''simple docstring'''







        return {
            "feature_size": self.feature_size,
            "padding_value": self.padding_value,
            "sampling_rate": self.sampling_rate,
            "return_attention_mask": self.return_attention_mask,
        }


    def 		snake_case__	( self					:			Optional[Any],lowercase_					:			str=False,lowercase_					:			Tuple=False						)->       List[str]:





        '''simple docstring'''







        def _flatten(lowercase_					:			Any						):
            return list(itertools.chain(*lowercase_						)						)

        if equal_length:
            A__        =  floats_list((self.batch_size, self.max_seq_length)						)
        else:
            # make sure that inputs increase in size
            A__        =  [
                _flatten(floats_list((x, self.feature_size)						)						)
                for x in range(self.min_seq_length,self.max_seq_length,self.seq_length_diff						)
            ]

        if numpify:
            A__        =  [np.asarray(lowercase_						) for x in audio_inputs]

        return audio_inputs




@require_torch
class 					A		(				_UpperCAmelCase       ,				unittest.TestCase					):







    """simple docstring"""





    lowerCamelCase						      =				EncodecFeatureExtractor


    def 		snake_case__	( self					:			List[str]						)->       List[Any]:





        '''simple docstring'''







        A__        =  EnCodecFeatureExtractionTester(self						)


    def 		snake_case__	( self					:			Optional[Any]						)->       List[str]:





        '''simple docstring'''







        A__        =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
        # create three inputs of length 800, 1000, and 1200
        A__        =  [floats_list((1, x)						)[0] for x in range(8_0_0,1_4_0_0,2_0_0						)]
        A__        =  [np.asarray(lowercase_						) for audio_input in audio_inputs]

        # Test not batched input
        A__        =  feat_extract(audio_inputs[0],return_tensors='np'						).input_values
        A__        =  feat_extract(np_audio_inputs[0],return_tensors='np'						).input_values
        self.assertTrue(np.allclose(lowercase_,lowercase_,atol=1E-3						)						)

        # Test batched
        A__        =  feat_extract(lowercase_,padding=lowercase_,return_tensors='np'						).input_values
        A__        =  feat_extract(lowercase_,padding=lowercase_,return_tensors='np'						).input_values
        for enc_seq_a, enc_seq_a in zip(lowercase_,lowercase_						):
            self.assertTrue(np.allclose(lowercase_,lowercase_,atol=1E-3						)						)


    def 		snake_case__	( self					:			Optional[Any]						)->       Optional[Any]:





        '''simple docstring'''







        A__        =  self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
        A__        =  np.random.rand(1_0_0						).astype(np.floataa						)
        A__        =  np_audio_inputs.tolist()

        for inputs in [py_audio_inputs, np_audio_inputs]:
            A__        =  feature_extractor.pad([{'input_values': inputs}],return_tensors='np'						)
            self.assertTrue(np_processed.input_values.dtype == np.floataa						)
            A__        =  feature_extractor.pad([{'input_values': inputs}],return_tensors='pt'						)
            self.assertTrue(pt_processed.input_values.dtype == torch.floataa						)


    def 		snake_case__	( self					:			int,lowercase_					:			Tuple						)->       Dict:





        '''simple docstring'''







        from datasets import load_dataset

        A__        =  load_dataset('hf-internal-testing/librispeech_asr_dummy','clean',split='validation'						)
        # automatic decoding with librispeech
        A__        =  ds.sort('id'						).select(range(lowercase_						)						)[:num_samples]['audio']

        return [x["array"] for x in audio_samples]


    def 		snake_case__	( self					:			int						)->       List[str]:





        '''simple docstring'''







        A__        =  torch.tensor(
            [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
             3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
             2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
             4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
             7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
             4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]						)
        # fmt: on
        A__        =  self._load_datasamples(1						)
        A__        =  EncodecFeatureExtractor()
        A__        =  feature_extractor(lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(input_values.shape,(1, 1, 9_3_6_8_0)						)
        self.assertTrue(torch.allclose(input_values[0, 0, :3_0],lowercase_,atol=1E-6						)						)


    def 		snake_case__	( self					:			int						)->       Optional[Any]:





        '''simple docstring'''







        A__        =  torch.tensor(
            [2.3_804E-03, 2.0_752E-03, 1.9_836E-03, 2.1_057E-03, 1.6_174E-03,
             3.0_518E-04, 9.1_553E-05, 3.3_569E-04, 9.7_656E-04, 1.8_311E-03,
             2.0_142E-03, 2.1_057E-03, 1.7_395E-03, 4.5_776E-04, -3.9_673E-04,
             4.5_776E-04, 1.0_071E-03, 9.1_553E-05, 4.8_828E-04, 1.1_597E-03,
             7.3_242E-04, 9.4_604E-04, 1.8_005E-03, 1.8_311E-03, 8.8_501E-04,
             4.2_725E-04, 4.8_828E-04, 7.3_242E-04, 1.0_986E-03, 2.1_057E-03]						)
        # fmt: on
        A__        =  self._load_datasamples(1						)
        A__        =  [np.tile(input_audio[0][None],reps=(2, 1)						)]
        input_audio[0][1] *= 0.5
        A__        =  EncodecFeatureExtractor(feature_size=2						)
        A__        =  feature_extractor(lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(input_values.shape,(1, 2, 9_3_6_8_0)						)
        self.assertTrue(torch.allclose(input_values[0, 0, :3_0],lowercase_,atol=1E-6						)						)
        self.assertTrue(torch.allclose(input_values[0, 1, :3_0],EXPECTED_INPUT_VALUES * 0.5,atol=1E-6						)						)


    def 		snake_case__	( self					:			Any						)->       int:





        '''simple docstring'''







        A__        =  self._load_datasamples(2						)
        # would be easier if the stride was like
        A__        =  EncodecFeatureExtractor(feature_size=1,chunk_length_s=1,overlap=0.01						)

        # pad and trunc raise an error ?
        with self.assertRaisesRegex(
            lowercase_,'^Both padding and truncation were set. Make sure you only set one.$',):
            A__        =  feature_extractor(
                lowercase_,padding='max_length',truncation=lowercase_,return_tensors='pt'						).input_values

        # truncate to chunk
        A__        =  feature_extractor(lowercase_,truncation=lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(2, 1, 7_1_5_2_0)						)  # 2 chunks

        # force truncate to max_length
        A__        =  feature_extractor(
            lowercase_,truncation=lowercase_,max_length=4_8_0_0_0,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(2, 1, 4_8_0_0_0)						)

        # pad to chunk
        A__        =  feature_extractor(lowercase_,padding=lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(padded_outputs.shape,(2, 1, 9_5_2_8_0)						)

        # pad to chunk
        A__        =  feature_extractor(lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(2, 1, 9_5_2_8_0)						)

        # force pad to max length
        A__        =  feature_extractor(
            lowercase_,padding='max_length',max_length=1_0_0_0_0_0,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(2, 1, 1_0_0_0_0_0)						)

        # force no pad
        with self.assertRaisesRegex(
            lowercase_,'^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$',):
            A__        =  feature_extractor(lowercase_,padding=lowercase_,return_tensors='pt'						).input_values

        A__        =  feature_extractor(input_audio[0],padding=lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(1, 1, 9_3_6_8_0)						)

        # no pad if no chunk_length_s
        A__        =  None
        with self.assertRaisesRegex(
            lowercase_,'^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$',):
            A__        =  feature_extractor(lowercase_,padding=lowercase_,return_tensors='pt'						).input_values

        A__        =  feature_extractor(input_audio[0],padding=lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(1, 1, 9_3_6_8_0)						)

        # no pad if no overlap
        A__        =  2
        A__        =  None
        with self.assertRaisesRegex(
            lowercase_,'^Unable to create tensor, you should probably activate padding with \'padding=True\' to have batched tensors with the same length.$',):
            A__        =  feature_extractor(lowercase_,padding=lowercase_,return_tensors='pt'						).input_values

        A__        =  feature_extractor(input_audio[0],padding=lowercase_,return_tensors='pt'						).input_values
        self.assertEquals(truncated_outputs.shape,(1, 1, 9_3_6_8_0)						)

