





"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_snake_case	:						Tuple					=					logging.get_logger(__name__)

_snake_case	:						List[Any]					=					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case	:						Optional[int]					=					{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

_snake_case	:						int					=					{
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}






class    _UpperCAmelCase  (      _UpperCAmelCase					):
					UpperCamelCase						=				VOCAB_FILES_NAMES
					UpperCamelCase						=				PRETRAINED_VOCAB_FILES_MAP
					UpperCamelCase						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCamelCase						=				["""input_ids""", """attention_mask"""]
					UpperCamelCase						=				LongformerTokenizer


					def __init__(   self     :Tuple	,			__UpperCamelCase     :Union[str, Any]=None	,			__UpperCamelCase     :Optional[Any]=None	,			__UpperCamelCase     :str=None	,			__UpperCamelCase     :Union[str, Any]="replace"	,			__UpperCamelCase     :List[Any]="<s>"	,			__UpperCamelCase     :Optional[Any]="</s>"	,			__UpperCamelCase     :str="</s>"	,			__UpperCamelCase     :Union[str, Any]="<s>"	,			__UpperCamelCase     :Any="<unk>"	,			__UpperCamelCase     :Union[str, Any]="<pad>"	,			__UpperCamelCase     :int="<mask>"	,			__UpperCamelCase     :Optional[int]=False	,			__UpperCamelCase     :int=True	,			**__UpperCamelCase     :Any	,			):
												super().__init__(
												    snake_case_	,			snake_case_	,			tokenizer_file=snake_case_	,			errors=snake_case_	,			bos_token=snake_case_	,			eos_token=snake_case_	,			sep_token=snake_case_	,			cls_token=snake_case_	,			unk_token=snake_case_	,			pad_token=snake_case_	,			mask_token=snake_case_	,			add_prefix_space=snake_case_	,			trim_offsets=snake_case_	,			**snake_case_	,			)

												A						=    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
												if pre_tok_state.get("add_prefix_space"	,			snake_case_						) != add_prefix_space:
																			A						=    getattr(snake_case_	,			pre_tok_state.pop("type"						)						)
																			A						=    add_prefix_space
																			A						=    pre_tok_class(**snake_case_						)

												A						=    add_prefix_space

												A						=    "post_processor"
												A						=    getattr(self.backend_tokenizer	,			snake_case_	,			snake_case_						)
												if tokenizer_component_instance:
																			A						=    json.loads(tokenizer_component_instance.__getstate__()						)

																			# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																			if "sep" in state:
																										A						=    tuple(state["sep"]						)
																			if "cls" in state:
																										A						=    tuple(state["cls"]						)

																			A						=    False

																			if state.get("add_prefix_space"	,			snake_case_						) != add_prefix_space:
																										A						=    add_prefix_space
																										A						=    True

																			if state.get("trim_offsets"	,			snake_case_						) != trim_offsets:
																										A						=    trim_offsets
																										A						=    True

																			if changes_to_apply:
																										A						=    getattr(snake_case_	,			state.pop("type"						)						)
																										A						=    component_class(**snake_case_						)
																										setattr(self.backend_tokenizer	,			snake_case_	,			snake_case_						)


					@property
					def       lowerCamelCase			(   self     :Any						):
												if self._mask_token is None:
																			if self.verbose:
																										logger.error("Using mask_token, but it is not set yet."						)
																			return None
												return str(self._mask_token						)


					@mask_token.setter
					def       lowerCamelCase			(   self     :Optional[int]	,			__UpperCamelCase     :List[Any]						):
												A						=    AddedToken(snake_case_	,			lstrip=snake_case_	,			rstrip=snake_case_						) if isinstance(snake_case_	,			snake_case_						) else value
												A						=    value


					def       lowerCamelCase			(   self     :Optional[int]	,			*__UpperCamelCase     :int	,			**__UpperCamelCase     :Dict						):
												A						=    kwargs.get("is_split_into_words"	,			snake_case_						)
												assert self.add_prefix_space or not is_split_into_words, (
												    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*snake_case_	,			**snake_case_						)


					def       lowerCamelCase			(   self     :List[str]	,			*__UpperCamelCase     :Optional[Any]	,			**__UpperCamelCase     :Any						):
												A						=    kwargs.get("is_split_into_words"	,			snake_case_						)

												assert self.add_prefix_space or not is_split_into_words, (
												    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*snake_case_	,			**snake_case_						)


					def       lowerCamelCase			(   self     :Optional[int]	,			__UpperCamelCase     :List[str]	,			__UpperCamelCase     :Tuple = None						):
												A						=    self._tokenizer.model.save(snake_case_	,			name=snake_case_						)
												return tuple(snake_case_						)


					def       lowerCamelCase			(   self     :Dict	,			__UpperCamelCase     :Any	,			__UpperCamelCase     :Dict=None						):
												A						=    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
												if token_ids_a is None:
																			return output

												return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


					def       lowerCamelCase			(   self     :Dict	,			__UpperCamelCase     :int	,			__UpperCamelCase     :List[Any] = None						):
												A						=    [self.sep_token_id]
												A						=    [self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep						) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]



'''simple docstring'''

# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


A_	   = {
    "configuration_convnextv2": [
        "CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ConvNextV2Config",
    ]
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_	   = [
						    "CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST",
						    "ConvNextV2ForImageClassification",
						    "ConvNextV2Model",
						    "ConvNextV2PreTrainedModel",
						    "ConvNextV2Backbone",
						]


if TYPE_CHECKING:
						from .configuration_convnextva import (
						    CONVNEXTV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    ConvNextVaConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_convnextva import (
												    CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST,
												    ConvNextVaBackbone,
												    ConvNextVaForImageClassification,
												    ConvNextVaModel,
												    ConvNextVaPreTrainedModel,
												)

else:
						import sys

						A_	   = _LazyModule(__name__, globals()["__file__"], _import_structure)





