




import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    def __init__(			self      :	List[str]					, UpperCAmelCase__      :	str = "▁"					, UpperCAmelCase__      :	bool = True					, UpperCAmelCase__      :	Union[str, AddedToken] = "<unk>"					, UpperCAmelCase__      :	Union[str, AddedToken] = "</s>"					, UpperCAmelCase__      :	Union[str, AddedToken] = "<pad>"					, )						->Optional[Any]:





           '''simple docstring'''

           A__       =    {
               '''pad''': {'''id''': 0, '''token''': pad_token},
               '''eos''': {'''id''': 1, '''token''': eos_token},
               '''unk''': {'''id''': 2, '''token''': unk_token},
           }

           A__       =    [None] * len(self.special_tokens)
           for token_dict in self.special_tokens.values():
                  A__       =    token_dict['''token''']

           A__       =    Tokenizer(Unigram())

           A__       =    normalizers.Sequence(
               [
                   normalizers.Nmt(),
                   normalizers.NFKC(),
                   normalizers.Replace(Regex(''' {2,}''')					, ''' '''),
                   normalizers.Lowercase(),
               ])
           A__       =    pre_tokenizers.Sequence(
               [
                   pre_tokenizers.Metaspace(replacement=UpperCAmelCase__					, add_prefix_space=UpperCAmelCase__),
                   pre_tokenizers.Digits(individual_digits=UpperCAmelCase__),
                   pre_tokenizers.Punctuation(),
               ])
           A__       =    decoders.Metaspace(replacement=UpperCAmelCase__					, add_prefix_space=UpperCAmelCase__)

           A__       =    TemplateProcessing(
               single=f"""$A {self.special_tokens["eos"]["token"]}"""					, special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]					, )

           A__       =    {
               '''model''': '''SentencePieceUnigram''',
               '''replacement''': replacement,
               '''add_prefix_space''': add_prefix_space,
           }

           super().__init__(UpperCAmelCase__					, UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Any					, UpperCAmelCase__      :	Union[str, List[str]]					, UpperCAmelCase__      :	int = 8_000					, UpperCAmelCase__      :	bool = True					, )						->Tuple:





           '''simple docstring'''

           A__       =    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase__					, special_tokens=self.special_tokens_list					, show_progress=UpperCAmelCase__					, )

           if isinstance(UpperCAmelCase__					, UpperCAmelCase__):
                  A__       =    [files]
           self._tokenizer.train(UpperCAmelCase__					, trainer=UpperCAmelCase__)

           self.add_unk_id()


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple					, UpperCAmelCase__      :	Union[Iterator[str], Iterator[Iterator[str]]]					, UpperCAmelCase__      :	int = 8_000					, UpperCAmelCase__      :	bool = True					, )						->List[Any]:





           '''simple docstring'''

           A__       =    trainers.UnigramTrainer(
               vocab_size=UpperCAmelCase__					, special_tokens=self.special_tokens_list					, show_progress=UpperCAmelCase__					, )

           self._tokenizer.train_from_iterator(UpperCAmelCase__					, trainer=UpperCAmelCase__)

           self.add_unk_id()





    def        SCREAMING_SNAKE_CASE							(			self      :	Any)						->str:





           '''simple docstring'''

           A__       =    json.loads(self._tokenizer.to_str())

           A__       =    self.special_tokens['''unk''']['''id''']

           A__       =    Tokenizer.from_str(json.dumps(UpperCAmelCase__))








import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase	:		Tuple							       =   logging.get_logger(__name__)

_lowerCamelCase	:		str							       =   {
    """microsoft/git-base""": """https://huggingface.co/microsoft/git-base/resolve/main/config.json""",
}


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      '''git_vision_model'''


    def __init__(			self      :	Any					, UpperCAmelCase__      :	Any=768					, UpperCAmelCase__      :	int=3_072					, UpperCAmelCase__      :	List[str]=12					, UpperCAmelCase__      :	Dict=12					, UpperCAmelCase__      :	Optional[int]=3					, UpperCAmelCase__      :	List[Any]=224					, UpperCAmelCase__      :	Union[str, Any]=16					, UpperCAmelCase__      :	Union[str, Any]="quick_gelu"					, UpperCAmelCase__      :	Dict=1e-5					, UpperCAmelCase__      :	Union[str, Any]=0.0					, UpperCAmelCase__      :	Any=0.02					, **UpperCAmelCase__      :	Any					, )						->Optional[int]:





           '''simple docstring'''

           super().__init__(**UpperCAmelCase__)

           A__       =    hidden_size
           A__       =    intermediate_size
           A__       =    num_hidden_layers
           A__       =    num_attention_heads
           A__       =    num_channels
           A__       =    patch_size
           A__       =    image_size
           A__       =    initializer_range
           A__       =    attention_dropout
           A__       =    layer_norm_eps
           A__       =    hidden_act





    @classmethod
    def        SCREAMING_SNAKE_CASE							(			cls      :	Any					, UpperCAmelCase__      :	Union[str, os.PathLike]					, **UpperCAmelCase__      :	int)						->"PretrainedConfig":





           '''simple docstring'''

           cls._set_token_in_kwargs(UpperCAmelCase__)

           A__ ,							A__       =    cls.get_config_dict(UpperCAmelCase__					, **UpperCAmelCase__)

           # get the vision config dict if we are loading from GITConfig
           if config_dict.get('''model_type''') == "git":
                  A__       =    config_dict['''vision_config''']

           if "model_type" in config_dict and hasattr(cls					, '''model_type''') and config_dict["model_type"] != cls.model_type:
                  logger.warning(
                      f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
                      f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors.""")

           return cls.from_dict(UpperCAmelCase__					, **UpperCAmelCase__)


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      '''git'''


    def __init__(			self      :	Dict					, UpperCAmelCase__      :	Dict=None					, UpperCAmelCase__      :	int=30_522					, UpperCAmelCase__      :	Optional[int]=768					, UpperCAmelCase__      :	Dict=6					, UpperCAmelCase__      :	int=12					, UpperCAmelCase__      :	List[str]=3_072					, UpperCAmelCase__      :	str="gelu"					, UpperCAmelCase__      :	int=0.1					, UpperCAmelCase__      :	Union[str, Any]=0.1					, UpperCAmelCase__      :	List[Any]=1_024					, UpperCAmelCase__      :	List[str]=0.02					, UpperCAmelCase__      :	Any=1e-12					, UpperCAmelCase__      :	Union[str, Any]=0					, UpperCAmelCase__      :	List[Any]="absolute"					, UpperCAmelCase__      :	int=True					, UpperCAmelCase__      :	Any=False					, UpperCAmelCase__      :	int=101					, UpperCAmelCase__      :	Tuple=102					, UpperCAmelCase__      :	Dict=None					, **UpperCAmelCase__      :	List[str]					, )						->Any:





           '''simple docstring'''

           super().__init__(bos_token_id=UpperCAmelCase__					, eos_token_id=UpperCAmelCase__					, pad_token_id=UpperCAmelCase__					, **UpperCAmelCase__)

           if vision_config is None:
                  A__       =    {}
                  logger.info('''vision_config is None. initializing the GitVisionConfig with default values.''')

           A__       =    GitVisionConfig(**UpperCAmelCase__)
           A__       =    vocab_size
           A__       =    hidden_size
           A__       =    num_hidden_layers
           A__       =    num_attention_heads
           A__       =    hidden_act
           A__       =    intermediate_size
           A__       =    hidden_dropout_prob
           A__       =    attention_probs_dropout_prob
           A__       =    max_position_embeddings
           A__       =    initializer_range
           A__       =    layer_norm_eps
           A__       =    position_embedding_type
           A__       =    use_cache
           A__       =    tie_word_embeddings
           A__       =    num_image_with_embedding

           A__       =    bos_token_id
           A__       =    eos_token_id





    def        SCREAMING_SNAKE_CASE							(			self      :	Any)						->List[Any]:





           '''simple docstring'''

           A__       =    copy.deepcopy(self.__dict__)
           A__       =    self.vision_config.to_dict()
           A__       =    self.__class__.model_type
           return output


