

from io import BytesIO
from typing import List, Union

import requests

from ..utils import add_end_docstrings, is_decord_available, is_torch_available, logging, requires_backends
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_decord_available():
     import numpy as np
     from decord import VideoReader


if is_torch_available():
     from ..models.auto.modeling_auto import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING

lowercase_       		=  logging.get_logger(__name__)


@add_end_docstrings(_UpperCAmelCase )
class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    def __init__(      self :       List[str],*lowercase_ :       Dict,**lowercase_ :       List[Any]		)->    Dict:





        '''simple docstring'''







        super().__init__(*lowercase_,**lowercase_		)
        requires_backends(self,'decord'		)
        self.check_model_type(lowercase_		)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       List[str]=None,lowercase_ :       int=None,lowercase_ :       List[Any]=None		)->    List[Any]:





        '''simple docstring'''







        A__          =   {}
        if frame_sampling_rate is not None:
            A__          =   frame_sampling_rate
        if num_frames is not None:
            A__          =   num_frames

        A__          =   {}
        if top_k is not None:
            A__          =   top_k
        return preprocess_params, {}, postprocess_params


    def __call__(      self :       List[Any],lowercase_ :       Union[str, List[str]],**lowercase_ :       str		)->    Any:





        '''simple docstring'''







        return super().__call__(lowercase_,**lowercase_		)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       Union[str, Any],lowercase_ :       List[str]=None,lowercase_ :       Tuple=1		)->    int:





        '''simple docstring'''







        if num_frames is None:
            A__          =   self.model.config.num_frames

        if video.startswith('http://'		) or video.startswith('https://'		):
            A__          =   BytesIO(requests.get(lowercase_		).content		)

        A__          =   VideoReader(lowercase_		)
        videoreader.seek(0		)

        A__          =   0
        A__          =   num_frames * frame_sampling_rate - 1
        A__          =   np.linspace(lowercase_,lowercase_,num=lowercase_,dtype=np.intaa		)

        A__          =   videoreader.get_batch(lowercase_		).asnumpy()
        A__          =   list(lowercase_		)

        A__          =   self.image_processor(lowercase_,return_tensors=self.framework		)
        return model_inputs


    def 						snake_case__     (      self :       Optional[int],lowercase_ :       Tuple		)->    Any:





        '''simple docstring'''







        A__          =   self.model(**lowercase_		)
        return model_outputs


    def 						snake_case__     (      self :       List[Any],lowercase_ :       List[str],lowercase_ :       Union[str, Any]=5		)->    str:





        '''simple docstring'''







        if top_k > self.model.config.num_labels:
            A__          =   self.model.config.num_labels

        if self.framework == "pt":
            A__          =   model_outputs.logits.softmax(-1		)[0]
            A__      ,						A__          =   probs.topk(lowercase_		)
        else:
            raise ValueError(F'Unsupported framework: {self.framework}'		)

        A__          =   scores.tolist()
        A__          =   ids.tolist()
        return [{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(lowercase_,lowercase_		)]




import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_       		=  get_tests_dir("fixtures/test_sentencepiece.model")


@require_sentencepiece
@require_tokenizers
class  A						(							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							ReformerTokenizer
    lowerCamelCase			=							ReformerTokenizerFast
    lowerCamelCase			=							True
    lowerCamelCase			=							False
    lowerCamelCase			=							True


    def 						snake_case__     (      self :       Any		)->    str:





        '''simple docstring'''







        super().setUp()

        A__          =   ReformerTokenizer(lowercase_,keep_accents=lowercase_		)
        tokenizer.save_pretrained(self.tmpdirname		)


    def 						snake_case__     (      self :       Optional[int]		)->    Optional[int]:





        '''simple docstring'''







        A__          =   '<s>'
        A__          =   1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_		),lowercase_		)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_		),lowercase_		)


    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   list(self.get_tokenizer().get_vocab().keys()		)

        self.assertEqual(vocab_keys[0],'<unk>'		)
        self.assertEqual(vocab_keys[1],'<s>'		)
        self.assertEqual(vocab_keys[-1],'j'		)
        self.assertEqual(len(lowercase_		),1_0_0_0		)


    def 						snake_case__     (      self :       Dict		)->    Dict:





        '''simple docstring'''







        self.assertEqual(self.get_tokenizer().vocab_size,1_0_0_0		)


    def 						snake_case__     (      self :       Dict		)->    List[str]:





        '''simple docstring'''







        if not self.test_rust_tokenizer:
            return

        A__          =   self.get_tokenizer()
        A__          =   self.get_rust_tokenizer()

        A__          =   'I was born in 92000, and this is falsé.'

        A__          =   tokenizer.tokenize(lowercase_		)
        A__          =   rust_tokenizer.tokenize(lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)

        A__          =   tokenizer.encode(lowercase_,add_special_tokens=lowercase_		)
        A__          =   rust_tokenizer.encode(lowercase_,add_special_tokens=lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)

        A__          =   self.get_rust_tokenizer()
        A__          =   tokenizer.encode(lowercase_		)
        A__          =   rust_tokenizer.encode(lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)


    def 						snake_case__     (      self :       int,lowercase_ :       Optional[int]=1_5		)->    Optional[Any]:





        '''simple docstring'''







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                A__          =   self.rust_tokenizer_class.from_pretrained(lowercase_,**lowercase_		)

                # Simple input
                A__          =   'This is a simple input'
                A__          =   ['This is a simple input 1', 'This is a simple input 2']
                A__          =   ('This is a simple input', 'This is a pair')
                A__          =   [
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(lowercase_,tokenizer_r.encode,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Simple input
                self.assertRaises(lowercase_,tokenizer_r.encode_plus,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Simple input
                self.assertRaises(
                    lowercase_,tokenizer_r.batch_encode_plus,lowercase_,max_length=lowercase_,padding='max_length',)

                # Pair input
                self.assertRaises(lowercase_,tokenizer_r.encode,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Pair input
                self.assertRaises(lowercase_,tokenizer_r.encode_plus,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Pair input
                self.assertRaises(
                    lowercase_,tokenizer_r.batch_encode_plus,lowercase_,max_length=lowercase_,padding='max_length',)


    def 						snake_case__     (      self :       List[Any]		)->    Tuple:





        '''simple docstring'''







        pass


    def 						snake_case__     (      self :       Dict		)->    str:





        '''simple docstring'''







        A__          =   ReformerTokenizer(lowercase_,keep_accents=lowercase_		)

        A__          =   tokenizer.tokenize('This is a test'		)
        self.assertListEqual(lowercase_,['▁This', '▁is', '▁a', '▁t', 'est']		)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(lowercase_		),[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2],)

        A__          =   tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
        self.assertListEqual(
            lowercase_,[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ],)
        A__          =   tokenizer.convert_tokens_to_ids(lowercase_		)
        self.assertListEqual(
            lowercase_,[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4],)

        A__          =   tokenizer.convert_ids_to_tokens(lowercase_		)
        self.assertListEqual(
            lowercase_,[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ],)


    @cached_property
    def 						snake_case__     (      self :       Optional[int]		)->    Any:





        '''simple docstring'''







        return ReformerTokenizer.from_pretrained('google/reformer-crime-and-punishment'		)


    @slow
    def 						snake_case__     (      self :       str		)->    Tuple:





        '''simple docstring'''







        A__          =   'Hello World!'
        A__          =   [1_2_6, 3_2, 2_6_2, 1_5_2, 3_8, 7_2, 2_8_7]

        self.assertListEqual(lowercase_,self.big_tokenizer.encode(lowercase_		)		)


    @slow
    def 						snake_case__     (      self :       Optional[int]		)->    str:





        '''simple docstring'''







        A__          =   (
            'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
            ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
        )
        A__          =   [
            1_0_8,
            2_6_5,
            2_4,
            1_1_1,
            4,
            2_5_8,
            1_5_6,
            3_5,
            2_8,
            2_7_5,
            3,
            2_5_9,
            2_9_7,
            2_6_0,
            8_4,
            4,
            3_5,
            1_1_0,
            4_4,
            8,
            2_5_9,
            9_1,
            2_6_8,
            2_1,
            1_1,
            2_0_9,
            2_7_4,
            1_0_9,
            2_6_6,
            2_7_7,
            1_1_7,
            8_6,
            9_3,
            3_1_5,
            2_5_8,
            2_7_8,
            2_5_8,
            2_7_7,
            2_5_8,
            0,
            2_5_8,
            2_8_8,
            2_5_8,
            3_1_9,
            2_5_8,
            0,
            2_5_8,
            0,
            2_5_8,
            0,
            2_5_8,
            0,
            2_5_8,
            2_8_7,
            2_5_8,
            3_1_5,
            2_5_8,
            2_8_9,
            2_5_8,
            2_7_8,
            9_9,
            2_6_9,
            2_6_6,
            2_6_2,
            8,
            2_5_9,
            2_4_1,
            4,
            2_1_7,
            2_3_0,
            2_6_8,
            2_6_6,
            5_5,
            1_6_8,
            1_0_6,
            7_5,
            1_9_3,
            2_6_6,
            2_2_3,
            2_7,
            4_9,
            2_6,
            2_8_2,
            2_5,
            2_6_4,
            2_9_9,
            1_9,
            2_6,
            0,
            2_5_8,
            2_7_7,
            1_1_7,
            8_6,
            9_3,
            1_7_6,
            1_8_3,
            2_7_0,
            1_1,
            2_6_2,
            4_2,
            6_1,
            2_6_5,
        ]

        self.assertListEqual(lowercase_,self.big_tokenizer.encode(lowercase_		)		)


    @require_torch
    @slow
    def 						snake_case__     (      self :       int		)->    Any:





        '''simple docstring'''







        import torch

        from transformers import ReformerConfig, ReformerModel

        # Build sequence
        A__          =   list(self.big_tokenizer.get_vocab().keys()		)[:1_0]
        A__          =   ' '.join(lowercase_		)
        A__          =   self.big_tokenizer.encode_plus(lowercase_,return_tensors='pt'		)
        A__          =   self.big_tokenizer.batch_encode_plus([sequence, sequence],return_tensors='pt'		)

        A__          =   ReformerConfig()
        # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
        A__          =   encoded_sequence['input_ids'].shape
        A__          =   ReformerModel(lowercase_		)

        # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
            model(**lowercase_		)
            model(**lowercase_		)


    @slow
    def 						snake_case__     (      self :       int		)->    Tuple:





        '''simple docstring'''







        A__          =   {'input_ids': [[1_0_8, 2_6_5, 2_4, 1_1_1, 4, 2_5_8, 1_5_6, 7, 5_1, 2_7_9, 5_8, 7, 7_6, 2_5, 6_9, 2_7_8], [1_4_0, 2_4_3, 2_6_4, 1_3_4, 1_7, 2_6_7, 7_7, 2_6_3, 2_2, 2_6_2, 2_9_7, 2_5_8, 3_0_4, 1_7_7, 2_7_9, 2_6_6, 1_4, 8_9, 1_3, 3_5, 2_6_1, 2_9_9, 2_7_2, 1_3_7, 2_7_5, 2_7_8]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        # This tokenizer does not know some characters like ")".
        # That is the reason why we use very simple texts here.
        # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
        A__          =   [
            'This is a very simple sentence.',
            'The quick brown fox jumps over the lazy dog.',
        ]

        self.tokenizer_integration_test_util(
            expected_encoding=lowercase_,model_name='google/reformer-crime-and-punishment',revision='0e6c3decb8211d49bf881013425dc8b0448b3f5a',padding=lowercase_,sequences=lowercase_,)

