

"""simple docstring"""




import random
import shutil
import tempfile
import unittest

import numpy as np

from transformers import TaTokenizer, TaTokenizerFast
from transformers.testing_utils import require_sentencepiece, require_torch
from transformers.utils.import_utils import is_speech_available, is_torch_available


if is_torch_available():
     pass

if is_speech_available():
     from transformers import EncodecFeatureExtractor, MusicgenProcessor


SCREAMING_SNAKE_CASE_  :      str							=      random.Random()



def 						UpperCAmelCase__ (					A__						,     A__=1.0						,     A__=None						,     A__=None )   ->   str:




   """simple docstring"""
   if rng is None:
      lowerCamelCase__						       =      global_rng

   lowerCamelCase__						       =      []
   for batch_idx in range(shape[0] ):
      values.append([] )
      for _ in range(shape[1] ):
         values[-1].append(rng.random() * scale )

   return values


@require_torch
@require_sentencepiece
class 		_A							(      unittest.TestCase       ):





      def        _lowerCamelCase	(	self		) ->       Dict:
         lowerCamelCase__						       =      "facebook/musicgen-small"
         lowerCamelCase__						       =      tempfile.mkdtemp()





      def        _lowerCamelCase	(	self     ,    **SCREAMING_SNAKE_CASE__		) ->       List[Any]:
         return TaTokenizer.from_pretrained(self.checkpoint     ,    **SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self     ,    **SCREAMING_SNAKE_CASE__		) ->       Any:
         return EncodecFeatureExtractor.from_pretrained(self.checkpoint     ,    **SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       str:
         shutil.rmtree(self.tmpdirname		)





      def        _lowerCamelCase	(	self		) ->       str:
         lowerCamelCase__						       =      self.get_tokenizer()
         lowerCamelCase__						       =      self.get_feature_extractor()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         processor.save_pretrained(self.tmpdirname		)
         lowerCamelCase__						       =      MusicgenProcessor.from_pretrained(self.tmpdirname		)

         self.assertEqual(processor.tokenizer.get_vocab()     ,    tokenizer.get_vocab()		)
         self.assertIsInstance(processor.tokenizer     ,    SCREAMING_SNAKE_CASE__		)

         self.assertEqual(processor.feature_extractor.to_json_string()     ,    feature_extractor.to_json_string()		)
         self.assertIsInstance(processor.feature_extractor     ,    SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       str:
         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=self.get_tokenizer()     ,    feature_extractor=self.get_feature_extractor()		)
         processor.save_pretrained(self.tmpdirname		)

         lowerCamelCase__						       =      self.get_tokenizer(bos_token="(BOS)"     ,    eos_token="(EOS)"		)
         lowerCamelCase__						       =      self.get_feature_extractor(do_normalize=SCREAMING_SNAKE_CASE__     ,    padding_value=1.0		)

         lowerCamelCase__						       =      MusicgenProcessor.from_pretrained(
             self.tmpdirname     ,    bos_token="(BOS)"     ,    eos_token="(EOS)"     ,    do_normalize=SCREAMING_SNAKE_CASE__     ,    padding_value=1.0		)

         self.assertEqual(processor.tokenizer.get_vocab()     ,    tokenizer_add_kwargs.get_vocab()		)
         self.assertIsInstance(processor.tokenizer     ,    SCREAMING_SNAKE_CASE__		)

         self.assertEqual(processor.feature_extractor.to_json_string()     ,    feature_extractor_add_kwargs.to_json_string()		)
         self.assertIsInstance(processor.feature_extractor     ,    SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       Union[str, Any]:
         lowerCamelCase__						       =      self.get_feature_extractor()
         lowerCamelCase__						       =      self.get_tokenizer()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      floats_list((3, 1000)		)

         lowerCamelCase__						       =      feature_extractor(SCREAMING_SNAKE_CASE__     ,    return_tensors="np"		)
         lowerCamelCase__						       =      processor(SCREAMING_SNAKE_CASE__     ,    return_tensors="np"		)

         for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum()     ,    input_processor[key].sum()     ,    delta=1e-2		)





      def        _lowerCamelCase	(	self		) ->       Optional[Any]:
         lowerCamelCase__						       =      self.get_feature_extractor()
         lowerCamelCase__						       =      self.get_tokenizer()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      "This is a test string"

         lowerCamelCase__						       =      processor(text=SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      tokenizer(SCREAMING_SNAKE_CASE__		)

         for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key]     ,    encoded_processor[key]		)





      def        _lowerCamelCase	(	self		) ->       Any:
         lowerCamelCase__						       =      self.get_feature_extractor()
         lowerCamelCase__						       =      self.get_tokenizer()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

         lowerCamelCase__						       =      processor.batch_decode(sequences=SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      tokenizer.batch_decode(SCREAMING_SNAKE_CASE__		)

         self.assertListEqual(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self		) ->       Tuple:
         lowerCamelCase__						       =      self.get_feature_extractor()
         lowerCamelCase__						       =      self.get_tokenizer()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         self.assertListEqual(
             processor.model_input_names     ,    feature_extractor.model_input_names     ,    msg="`processor` and `feature_extractor` model input names do not match"     ,    )





      def        _lowerCamelCase	(	self		) ->       List[str]:
         lowerCamelCase__						       =      self.get_feature_extractor(padding_side="left"		)
         lowerCamelCase__						       =      self.get_tokenizer()

         lowerCamelCase__						       =      MusicgenProcessor(tokenizer=SCREAMING_SNAKE_CASE__     ,    feature_extractor=SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      [floats_list((1, x)		)[0] for x in range(5     ,    20     ,    5		)]
         lowerCamelCase__						       =      processor(SCREAMING_SNAKE_CASE__		).padding_mask

         lowerCamelCase__						       =      np.asarray(floats_list((3, 20)		)		)[:, None, :]
         lowerCamelCase__						       =      processor.batch_decode(SCREAMING_SNAKE_CASE__     ,    padding_mask=SCREAMING_SNAKE_CASE__		)

         self.assertIsInstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)

         for audio in decoded_audios:
            self.assertIsInstance(SCREAMING_SNAKE_CASE__     ,    np.ndarray		)

         self.assertTrue(decoded_audios[0].shape == (1, 10)		)
         self.assertTrue(decoded_audios[1].shape == (1, 15)		)
         self.assertTrue(decoded_audios[2].shape == (1, 20)		)







"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE_  :      Optional[int]							=      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_  :      Optional[int]							=      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

SCREAMING_SNAKE_CASE_  :      Dict							=      {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

SCREAMING_SNAKE_CASE_  :      Optional[Any]							=      {
    '''gpt2''': 1024,
    '''gpt2-medium''': 1024,
    '''gpt2-large''': 1024,
    '''gpt2-xl''': 1024,
    '''distilgpt2''': 1024,
}



@lru_cache()
def 						UpperCAmelCase__ (					)   ->   Optional[int]:




   """simple docstring"""
   lowerCamelCase__						       =      (
       list(range(ord("!" )						,     ord("~" ) + 1 ) ) + list(range(ord("¡" )						,     ord("¬" ) + 1 ) ) + list(range(ord("®" )						,     ord("ÿ" ) + 1 ) )
   )
   lowerCamelCase__						       =      bs[:]
   lowerCamelCase__						       =      0
   for b in range(2**8 ):
      if b not in bs:
         bs.append(A__ )
         cs.append(2**8 + n )
         n += 1
   lowerCamelCase__						       =      [chr(A__ ) for n in cs]
   return dict(zip(A__						,     A__ ) )



def 						UpperCAmelCase__ (					A__ )   ->   int:




   """simple docstring"""
   lowerCamelCase__						       =      set()
   lowerCamelCase__						       =      word[0]
   for char in word[1:]:
      pairs.add((prev_char, char) )
      lowerCamelCase__						       =      char
   return pairs


class 		_A							(      __a       ):
      __a             =						VOCAB_FILES_NAMES
      __a             =						PRETRAINED_VOCAB_FILES_MAP
      __a             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a             =						['input_ids', 'attention_mask']





      def __init__(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__="replace"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__="<|endoftext|>"     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=False     ,    SCREAMING_SNAKE_CASE__=False     ,    **SCREAMING_SNAKE_CASE__     ,    ) ->       List[str]:
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else bos_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else eos_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else unk_token
         lowerCamelCase__						       =      AddedToken(SCREAMING_SNAKE_CASE__     ,    lstrip=SCREAMING_SNAKE_CASE__     ,    rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		) else pad_token
         super().__init__(
             errors=SCREAMING_SNAKE_CASE__     ,    unk_token=SCREAMING_SNAKE_CASE__     ,    bos_token=SCREAMING_SNAKE_CASE__     ,    eos_token=SCREAMING_SNAKE_CASE__     ,    pad_token=SCREAMING_SNAKE_CASE__     ,    add_prefix_space=SCREAMING_SNAKE_CASE__     ,    add_bos_token=SCREAMING_SNAKE_CASE__     ,    **SCREAMING_SNAKE_CASE__     ,    )
         lowerCamelCase__						       =      add_bos_token

         with open(SCREAMING_SNAKE_CASE__     ,    encoding="utf-8"		) as vocab_handle:
            lowerCamelCase__						       =      json.load(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      {v: k for k, v in self.encoder.items()}
         lowerCamelCase__						       =      errors  # how to handle errors in decoding
         lowerCamelCase__						       =      bytes_to_unicode()
         lowerCamelCase__						       =      {v: k for k, v in self.byte_encoder.items()}
         with open(SCREAMING_SNAKE_CASE__     ,    encoding="utf-8"		) as merges_handle:
            lowerCamelCase__						       =      merges_handle.read().split("\n"		)[1:-1]
         lowerCamelCase__						       =      [tuple(merge.split()		) for merge in bpe_merges]
         lowerCamelCase__						       =      dict(zip(SCREAMING_SNAKE_CASE__     ,    range(len(SCREAMING_SNAKE_CASE__		)		)		)		)
         lowerCamelCase__						       =      {}
         lowerCamelCase__						       =      add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowerCamelCase__						       =      re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"		)





      @property
      def        _lowerCamelCase	(	self		) ->       Optional[int]:
         return len(self.encoder		)





      def        _lowerCamelCase	(	self		) ->       str:
         return dict(self.encoder     ,    **self.added_tokens_encoder		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Any:
         if token in self.cache:
            return self.cache[token]
         lowerCamelCase__						       =      tuple(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      get_pairs(SCREAMING_SNAKE_CASE__		)

         if not pairs:
            return token

         while True:
            lowerCamelCase__						       =      min(SCREAMING_SNAKE_CASE__     ,    key=lambda SCREAMING_SNAKE_CASE__		: self.bpe_ranks.get(SCREAMING_SNAKE_CASE__     ,    float("inf"		)		)		)
            if bigram not in self.bpe_ranks:
               break
            lowerCamelCase__       ,							lowerCamelCase__						       =      bigram
            lowerCamelCase__						       =      []
            lowerCamelCase__						       =      0
            while i < len(SCREAMING_SNAKE_CASE__		):
               try:
                  lowerCamelCase__						       =      word.index(SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__		)
               except ValueError:
                  new_word.extend(word[i:]		)
                  break
               else:
                  new_word.extend(word[i:j]		)
                  lowerCamelCase__						       =      j

               if word[i] == first and i < len(SCREAMING_SNAKE_CASE__		) - 1 and word[i + 1] == second:
                  new_word.append(first + second		)
                  i += 2
               else:
                  new_word.append(word[i]		)
                  i += 1
            lowerCamelCase__						       =      tuple(SCREAMING_SNAKE_CASE__		)
            lowerCamelCase__						       =      new_word
            if len(SCREAMING_SNAKE_CASE__		) == 1:
               break
            else:
               lowerCamelCase__						       =      get_pairs(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      " ".join(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      word
         return word





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__=None		) ->       Optional[int]:
         if self.add_bos_token:
            lowerCamelCase__						       =      [self.bos_token_id]
         else:
            lowerCamelCase__						       =      []

         lowerCamelCase__						       =      bos_token_ids + token_ids_a

         if token_ids_a is None:
            return output

         return output + bos_token_ids + token_ids_a





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None     ,    SCREAMING_SNAKE_CASE__ = False		) ->       List[int]:
         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__     ,    token_ids_a=SCREAMING_SNAKE_CASE__     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

         if not self.add_bos_token:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE__     ,    token_ids_a=SCREAMING_SNAKE_CASE__     ,    already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

         if token_ids_a is None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		))
         return [1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__		))





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Tuple:
         lowerCamelCase__						       =      []
         for token in re.findall(self.pat     ,    SCREAMING_SNAKE_CASE__		):
            lowerCamelCase__						       =      "".join(
                self.byte_encoder[b] for b in token.encode("utf-8"		)		)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE__		).split(" "		)		)
         return bpe_tokens





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Tuple:
         return self.encoder.get(SCREAMING_SNAKE_CASE__     ,    self.encoder.get(self.unk_token		)		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       Any:
         return self.decoder.get(SCREAMING_SNAKE_CASE__		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       str:
         lowerCamelCase__						       =      "".join(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						       =      bytearray([self.byte_decoder[c] for c in text]		).decode("utf-8"     ,    errors=self.errors		)
         return text





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       Tuple[str]:
         if not os.path.isdir(SCREAMING_SNAKE_CASE__		):
            logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
            return
         lowerCamelCase__						       =      os.path.join(
             SCREAMING_SNAKE_CASE__     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
         lowerCamelCase__						       =      os.path.join(
             SCREAMING_SNAKE_CASE__     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

         with open(SCREAMING_SNAKE_CASE__     ,    "w"     ,    encoding="utf-8"		) as f:
            f.write(json.dumps(self.encoder     ,    indent=2     ,    sort_keys=SCREAMING_SNAKE_CASE__     ,    ensure_ascii=SCREAMING_SNAKE_CASE__		) + "\n"		)

         lowerCamelCase__						       =      0
         with open(SCREAMING_SNAKE_CASE__     ,    "w"     ,    encoding="utf-8"		) as writer:
            writer.write("#version: 0.2\n"		)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,    key=lambda SCREAMING_SNAKE_CASE__		: kv[1]		):
               if index != token_index:
                  logger.warning(
                      f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                      " Please check that the tokenizer is not corrupted!"		)
                  lowerCamelCase__						       =      token_index
               writer.write(" ".join(SCREAMING_SNAKE_CASE__		) + "\n"		)
               index += 1

         return vocab_file, merge_file





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__=False     ,    **SCREAMING_SNAKE_CASE__		) ->       int:
         lowerCamelCase__						       =      kwargs.pop("add_prefix_space"     ,    self.add_prefix_space		)
         if is_split_into_words or add_prefix_space:
            lowerCamelCase__						       =      " " + text
         return (text, kwargs)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__		) ->       List[int]:
         lowerCamelCase__						       =      []
         for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(SCREAMING_SNAKE_CASE__     ,    add_special_tokens=SCREAMING_SNAKE_CASE__		) + [self.eos_token_id]		)
         if len(SCREAMING_SNAKE_CASE__		) > self.model_max_length:
            lowerCamelCase__						       =      input_ids[-self.model_max_length :]
         return input_ids




