"""simple docstring"""







import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_SCREAMING_SNAKE_CASE	:		int   =   logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_SCREAMING_SNAKE_CASE	:		Tuple   =   direct_transformers_import(Path(__file__).parent)


_SCREAMING_SNAKE_CASE	:		List[Any]   =   {
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							Union[str, Any]    =    ["feature_extractor", "tokenizer"]
   # Names need to be attr_class for attr in attributes
   __UpperCAmelCase	:							Any    =    None
   __UpperCAmelCase	:							int    =    None
   __UpperCAmelCase	:							List[str]    =    None







   def __init__(	self ,				*_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      for key in kwargs:
         if key not in self.attributes:
            raise TypeError(F'''Unexpected keyword argument {key}.'''	)
      for arg, attribute_name in zip(_SCREAMING_SNAKE_CASE ,				self.attributes	):
         if attribute_name in kwargs:
            raise TypeError(F'''Got multiple values for argument {attribute_name}.'''	)
         else:
            __lowerCamelCase    		=					arg

      if len(_SCREAMING_SNAKE_CASE	) != len(self.attributes	):
         raise ValueError(
             F'''This processor requires {len(self.attributes	)} arguments: {', '.join(self.attributes	)}. Got '''
             F'''{len(_SCREAMING_SNAKE_CASE	)} arguments instead.'''	)

      # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
      for attribute_name, arg in kwargs.items():
         __lowerCamelCase    		=					getattr(self ,				F'''{attribute_name}_class'''	)
         # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
         __lowerCamelCase    		=					AUTO_TO_BASE_CLASS_MAPPING.get(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
         if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
            __lowerCamelCase    		=					tuple(getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) for n in class_name if n is not None	)
         else:
            __lowerCamelCase    		=					getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)

         if not isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
            raise ValueError(
                F'''Received a {type(_SCREAMING_SNAKE_CASE	).__name__} for argument {attribute_name}, but a {class_name} was expected.'''	)

         setattr(self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)







   def __repr__(	self	):


      '''simple docstring'''

      __lowerCamelCase    		=					[F'''- {name}: {repr(getattr(self ,				_SCREAMING_SNAKE_CASE	)	)}''' for name in self.attributes]
      __lowerCamelCase    		=					'\n'.join(_SCREAMING_SNAKE_CASE	)
      return F'''{self.__class__.__name__}:\n{attributes_repr}'''







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = False ,				**_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      os.makedirs(_SCREAMING_SNAKE_CASE ,				exist_ok=_SCREAMING_SNAKE_CASE	)

      if push_to_hub:
         __lowerCamelCase    		=					kwargs.pop('commit_message' ,				_SCREAMING_SNAKE_CASE	)
         __lowerCamelCase    		=					kwargs.pop('repo_id' ,				save_directory.split(os.path.sep	)[-1]	)
         __lowerCamelCase    		=					self._create_repo(_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)
         __lowerCamelCase    		=					self._get_files_timestamps(_SCREAMING_SNAKE_CASE	)
      # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
      # loaded from the Hub.
      if self._auto_class is not None:
         __lowerCamelCase    		=					[getattr(self ,				_SCREAMING_SNAKE_CASE	) for attribute_name in self.attributes]
         __lowerCamelCase    		=					[(a.init_kwargs if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) else a) for a in attrs]
         custom_object_save(self ,				_SCREAMING_SNAKE_CASE ,				config=_SCREAMING_SNAKE_CASE	)

      for attribute_name in self.attributes:
         __lowerCamelCase    		=					getattr(self ,				_SCREAMING_SNAKE_CASE	)
         # Include the processor class in the attribute config so this processor can then be reloaded with the
         # `AutoProcessor` API.
         if hasattr(_SCREAMING_SNAKE_CASE ,				'_set_processor_class'	):
            attribute._set_processor_class(self.__class__.__name__	)
         attribute.save_pretrained(_SCREAMING_SNAKE_CASE	)

      if self._auto_class is not None:
         # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
         for attribute_name in self.attributes:
            __lowerCamelCase    		=					getattr(self ,				_SCREAMING_SNAKE_CASE	)
            if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
               del attribute.init_kwargs["auto_map"]

      if push_to_hub:
         self._upload_modified_files(
             _SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				commit_message=_SCREAMING_SNAKE_CASE ,				token=kwargs.get('use_auth_token'	) ,				)







   @classmethod
   def    _A				(	cls ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = False ,				_SCREAMING_SNAKE_CASE = False ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = "main" ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      __lowerCamelCase    		=					cache_dir
      __lowerCamelCase    		=					force_download
      __lowerCamelCase    		=					local_files_only
      __lowerCamelCase    		=					revision

      __lowerCamelCase    		=					kwargs.pop('use_auth_token' ,				_SCREAMING_SNAKE_CASE	)
      if use_auth_token is not None:
         warnings.warn(
             'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.' ,				_SCREAMING_SNAKE_CASE	)
         if token is not None:
            raise ValueError(
                '`token` and `use_auth_token` are both specified. Please set only the argument `token`.'	)
         __lowerCamelCase    		=					use_auth_token

      if token is not None:
         # change to `token` in a follow-up PR
         __lowerCamelCase    		=					token

      __lowerCamelCase    		=					cls._get_arguments_from_pretrained(_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)
      return cls(*_SCREAMING_SNAKE_CASE	)







   @classmethod
   def    _A				(	cls ,				_SCREAMING_SNAKE_CASE="AutoProcessor"	):


      '''simple docstring'''

      if not isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
         __lowerCamelCase    		=					auto_class.__name__

      import transformers.models.auto as auto_module

      if not hasattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
         raise ValueError(F'''{auto_class} is not a valid auto class.'''	)

      __lowerCamelCase    		=					auto_class







   @classmethod
   def    _A				(	cls ,				_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      __lowerCamelCase    		=					[]
      for attribute_name in cls.attributes:
         __lowerCamelCase    		=					getattr(cls ,				F'''{attribute_name}_class'''	)
         if isinstance(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	):
            __lowerCamelCase    		=					tuple(getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) if n is not None else None for n in class_name	)
            __lowerCamelCase    		=					kwargs.get('use_fast' ,				_SCREAMING_SNAKE_CASE	)
            if use_fast and classes[1] is not None:
               __lowerCamelCase    		=					classes[1]
            else:
               __lowerCamelCase    		=					classes[0]
         else:
            __lowerCamelCase    		=					getattr(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)

         args.append(attribute_class.from_pretrained(_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)	)
      return args







   @property
   def    _A				(	self	):


      '''simple docstring'''

      __lowerCamelCase    		=					getattr(self ,				self.attributes[0]	)
      return getattr(_SCREAMING_SNAKE_CASE ,				'model_input_names' ,				_SCREAMING_SNAKE_CASE	)

_SCREAMING_SNAKE_CASE	:		Optional[int]   =   copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
     _SCREAMING_SNAKE_CASE	:		str   =   ProcessorMixin.push_to_hub.__doc__.format(
         object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
     )














import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
	UpperCAmelCase_					:		str           = None


UpperCAmelCase_					:		int           = logging.get_logger(__name__)

UpperCAmelCase_					:		List[str]           = {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase_					:		Any           = {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase_					:		Any           = {
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}

class        _SCREAMING_SNAKE_CASE			(     _a ):
	snake_case__				:	Dict	      =    VOCAB_FILES_NAMES
	snake_case__				:	Optional[int]	      =    PRETRAINED_VOCAB_FILES_MAP
	snake_case__				:	str	      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__				:	int	      =    ["""input_ids""", """attention_mask"""]
	snake_case__				:	Dict	      =    XLMRobertaTokenizer





	def __init__(						self							:   List[str]					,    __lowerCamelCase							:   List[Any]=None					,    __lowerCamelCase							:   int=None					,    __lowerCamelCase							:   Dict="<s>"					,    __lowerCamelCase							:   Optional[int]="</s>"					,    __lowerCamelCase							:   int="</s>"					,    __lowerCamelCase							:   Dict="<s>"					,    __lowerCamelCase							:   Optional[Any]="<unk>"					,    __lowerCamelCase							:   Union[str, Any]="<pad>"					,    __lowerCamelCase							:   Any="<mask>"					,    **__lowerCamelCase							:   List[str]					,    ):
		# Mask token behave like a normal word, i.e. include the space before it
		UpperCamelCase	:int		=      AddedToken(__lowerCamelCase					,    lstrip=__lowerCamelCase					,    rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase					,    __lowerCamelCase      ) else mask_token

		super().__init__(
		    __lowerCamelCase					,    tokenizer_file=__lowerCamelCase					,    bos_token=__lowerCamelCase					,    eos_token=__lowerCamelCase					,    sep_token=__lowerCamelCase					,    cls_token=__lowerCamelCase					,    unk_token=__lowerCamelCase					,    pad_token=__lowerCamelCase					,    mask_token=__lowerCamelCase					,    **__lowerCamelCase					,    )

		UpperCamelCase	:Optional[int]		=      vocab_file
		UpperCamelCase	:List[Any]		=      False if not self.vocab_file else True





	def 		_A					(						self							:   Dict					,    __lowerCamelCase							:   List[int]					,    __lowerCamelCase							:   Optional[List[int]] = None      ):

		if token_ids_a is None:
			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
		UpperCamelCase	:int		=      [self.cls_token_id]
		UpperCamelCase	:Any		=      [self.sep_token_id]
		return cls + token_ids_a + sep + sep + token_ids_a + sep





	def 		_A					(						self							:   List[str]					,    __lowerCamelCase							:   List[int]					,    __lowerCamelCase							:   Optional[List[int]] = None      ):
		UpperCamelCase	:str		=      [self.sep_token_id]
		UpperCamelCase	:int		=      [self.cls_token_id]

		if token_ids_a is None:
			return len(cls + token_ids_a + sep      ) * [0]
		return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]





	def 		_A					(						self							:   Optional[int]					,    __lowerCamelCase							:   str					,    __lowerCamelCase							:   Optional[str] = None      ):
		if not self.can_save_slow_tokenizer:
			raise ValueError(
			    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
			    """tokenizer."""      )

		if not os.path.isdir(__lowerCamelCase      ):
			logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""      )
			return
		UpperCamelCase	:Union[str, Any]		=      os.path.join(
		    __lowerCamelCase					,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

		if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowerCamelCase      ):
			copyfile(self.vocab_file					,    __lowerCamelCase      )

		return (out_vocab_file,)
