

import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class  A						(							_UpperCAmelCase    ,							unittest.TestCase ):







    """simple docstring"""





    lowerCamelCase			=							OpenAIGPTTokenizer
    lowerCamelCase			=							OpenAIGPTTokenizerFast
    lowerCamelCase			=							True
    lowerCamelCase			=							False


    def 						snake_case__     (      self :       List[Any]		)->    str:





        '''simple docstring'''







        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        A__          =   [
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            'w</w>',
            'r</w>',
            't</w>',
            'lo',
            'low',
            'er</w>',
            'low</w>',
            'lowest</w>',
            'newer</w>',
            'wider</w>',
            '<unk>',
        ]
        A__          =   dict(zip(lowercase_,range(len(lowercase_		)		)		)		)
        A__          =   ['#version: 0.2', 'l o', 'lo w', 'e r</w>', '']

        A__          =   os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['vocab_file']		)
        A__          =   os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['merges_file']		)
        with open(self.vocab_file,'w'		) as fp:
            fp.write(json.dumps(lowercase_		)		)
        with open(self.merges_file,'w'		) as fp:
            fp.write('\n'.join(lowercase_		)		)


    def 						snake_case__     (      self :       Optional[int],lowercase_ :       int		)->    Optional[int]:





        '''simple docstring'''







        return "lower newer", "lower newer"


    def 						snake_case__     (      self :       List[str]		)->    Union[str, Any]:





        '''simple docstring'''







        A__          =   OpenAIGPTTokenizer(self.vocab_file,self.merges_file		)

        A__          =   'lower'
        A__          =   ['low', 'er</w>']
        A__          =   tokenizer.tokenize(lowercase_		)
        self.assertListEqual(lowercase_,lowercase_		)

        A__          =   tokens + ['<unk>']
        A__          =   [1_4, 1_5, 2_0]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_		),lowercase_		)


    def 						snake_case__     (      self :       Optional[Any],lowercase_ :       Optional[int]=1_5		)->    str:





        '''simple docstring'''







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                A__          =   self.rust_tokenizer_class.from_pretrained(lowercase_,**lowercase_		)

                # Simple input
                A__          =   'This is a simple input'
                A__          =   ['This is a simple input 1', 'This is a simple input 2']
                A__          =   ('This is a simple input', 'This is a pair')
                A__          =   [
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(lowercase_,tokenizer_r.encode,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Simple input
                self.assertRaises(lowercase_,tokenizer_r.encode_plus,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Simple input
                self.assertRaises(
                    lowercase_,tokenizer_r.batch_encode_plus,lowercase_,max_length=lowercase_,padding='max_length',)

                # Pair input
                self.assertRaises(lowercase_,tokenizer_r.encode,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Pair input
                self.assertRaises(lowercase_,tokenizer_r.encode_plus,lowercase_,max_length=lowercase_,padding='max_length'		)

                # Pair input
                self.assertRaises(
                    lowercase_,tokenizer_r.batch_encode_plus,lowercase_,max_length=lowercase_,padding='max_length',)


    def 						snake_case__     (      self :       List[str]		)->    Optional[int]:





        '''simple docstring'''







        pass




@require_ftfy
@require_spacy
@require_tokenizers
class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    pass




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_       		=  logging.get_logger(__name__)

lowercase_       		=  {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    lowerCamelCase			=							'open-llama'


    def __init__(      self :       Any,lowercase_ :       Optional[int]=1_0_0_0_0_0,lowercase_ :       Union[str, Any]=4_0_9_6,lowercase_ :       Dict=1_1_0_0_8,lowercase_ :       Dict=3_2,lowercase_ :       Optional[int]=3_2,lowercase_ :       Dict="silu",lowercase_ :       Union[str, Any]=2_0_4_8,lowercase_ :       Optional[int]=0.02,lowercase_ :       Dict=1E-6,lowercase_ :       Dict=True,lowercase_ :       List[Any]=0,lowercase_ :       Optional[int]=1,lowercase_ :       str=2,lowercase_ :       str=False,lowercase_ :       str=True,lowercase_ :       int=0.1,lowercase_ :       List[Any]=0.1,lowercase_ :       List[Any]=True,lowercase_ :       Union[str, Any]=True,lowercase_ :       Any=None,**lowercase_ :       List[Any],)->    Tuple:





        '''simple docstring'''







        A__          =   vocab_size
        A__          =   max_position_embeddings
        A__          =   hidden_size
        A__          =   intermediate_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   hidden_act
        A__          =   initializer_range
        A__          =   rms_norm_eps
        A__          =   use_cache
        A__          =   kwargs.pop(
            'use_memorry_efficient_attention',lowercase_		)
        A__          =   hidden_dropout_prob
        A__          =   attention_dropout_prob
        A__          =   use_stable_embedding
        A__          =   shared_input_output_embedding
        A__          =   rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=lowercase_,bos_token_id=lowercase_,eos_token_id=lowercase_,tie_word_embeddings=lowercase_,**lowercase_,)


    def 						snake_case__     (      self :       str		)->    str:





        '''simple docstring'''







        if self.rope_scaling is None:
            return

        if not isinstance(self.rope_scaling,lowercase_		) or len(self.rope_scaling		) != 2:
            raise ValueError(
                '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                F'got {self.rope_scaling}'		)
        A__          =   self.rope_scaling.get('type',lowercase_		)
        A__          =   self.rope_scaling.get('factor',lowercase_		)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'		)
        if rope_scaling_factor is None or not isinstance(lowercase_,lowercase_		) or rope_scaling_factor <= 1.0:
            raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'		)

