




"""simple docstring"""






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import warnings
from typing import List
from unittest.mock import Mock

import torch
from torch.utils.data import DataLoader, IterableDataset, TensorDataset

from accelerate.accelerator import Accelerator
from accelerate.utils.dataclasses import DistributedType






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			def __init__(  self  ,      lowercase  ):
							_lowerCamelCase     :			Dict    =    data






			def __iter__(  self  ):
							for element in self.data:
											yield element







def      _snake_case			(				lowercase__=True		):
				_lowerCamelCase     :			Any    =    Accelerator(even_batches=lowercase__		)
				assert accelerator.num_processes == 2, "this script expects that two GPUs are available"
				return accelerator







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__							,    lowercase__ = False		):
				if iterable:
								_lowerCamelCase     :			List[Any]    =    DummyIterableDataset(torch.as_tensor(range(lowercase__		)		)		)
				else:
								_lowerCamelCase     :			List[str]    =    TensorDataset(torch.as_tensor(range(lowercase__		)		)		)

				_lowerCamelCase     :			Any    =    DataLoader(lowercase__							,    batch_size=lowercase__		)
				_lowerCamelCase     :			str    =    accelerator.prepare(lowercase__		)

				return dl







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__							,    lowercase__							,    lowercase__							,    ):
				_lowerCamelCase     :			int    =    create_dataloader(accelerator=lowercase__							,    dataset_size=lowercase__							,    batch_size=lowercase__		)

				_lowerCamelCase     :			Optional[Any]    =    [len(batch[0]		) for batch in dl]

				if accelerator.process_index == 0:
								assert batch_sizes == process_0_expected_batch_sizes
				elif accelerator.process_index == 1:
								assert batch_sizes == process_1_expected_batch_sizes







def      _snake_case			(				):
				_lowerCamelCase     :			int    =    create_accelerator()

				# without padding, we would expect a different number of batches
				verify_dataloader_batch_sizes(
				    lowercase__							,    dataset_size=3							,    batch_size=1							,    process_0_expected_batch_sizes=[1, 1]							,    process_1_expected_batch_sizes=[1, 1]							,    )

				# without padding, we would expect the same number of batches, but different sizes
				verify_dataloader_batch_sizes(
				    lowercase__							,    dataset_size=7							,    batch_size=2							,    process_0_expected_batch_sizes=[2, 2]							,    process_1_expected_batch_sizes=[2, 2]							,    )







def      _snake_case			(				):
				_lowerCamelCase     :			Optional[int]    =    create_accelerator(even_batches=lowercase__		)

				verify_dataloader_batch_sizes(
				    lowercase__							,    dataset_size=3							,    batch_size=1							,    process_0_expected_batch_sizes=[1, 1]							,    process_1_expected_batch_sizes=[1]							,    )

				verify_dataloader_batch_sizes(
				    lowercase__							,    dataset_size=7							,    batch_size=2							,    process_0_expected_batch_sizes=[2, 2]							,    process_1_expected_batch_sizes=[2, 1]							,    )







def      _snake_case			(				):
				_lowerCamelCase     :			int    =    create_accelerator(even_batches=lowercase__		)

				_lowerCamelCase     :			str    =    torch.nn.Linear(1							,    1		)
				_lowerCamelCase     :			Dict    =    accelerator.prepare(lowercase__		)

				_lowerCamelCase     :			Optional[Any]    =    create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1		)

				_lowerCamelCase     :			Optional[Any]    =    []
				with accelerator.join_uneven_inputs([ddp_model]		):
								for batch_idx, batch in enumerate(lowercase__		):
												_lowerCamelCase     :			str    =    ddp_model(batch[0].float()		)
												_lowerCamelCase     :			str    =    output.sum()
												loss.backward()
												batch_idxs.append(lowercase__		)

				accelerator.wait_for_everyone()

				if accelerator.process_index == 0:
								assert batch_idxs == [0, 1]
				elif accelerator.process_index == 1:
								assert batch_idxs == [0]







def      _snake_case			(				lowercase__		):
				with warnings.catch_warnings(record=lowercase__		) as w:
								with accelerator.join_uneven_inputs([Mock()]		):
												pass

								assert issubclass(w[-1].category							,    lowercase__		)
								assert "only supported for multi-GPU" in str(w[-1].message		)







def      _snake_case			(				):
				_lowerCamelCase     :			Any    =    True
				_lowerCamelCase     :			Any    =    False
				_lowerCamelCase     :			List[str]    =    create_accelerator(even_batches=lowercase__		)
				_lowerCamelCase     :			Dict    =    torch.nn.Linear(1							,    1		)
				_lowerCamelCase     :			Union[str, Any]    =    accelerator.prepare(lowercase__		)
				_lowerCamelCase     :			Any    =    create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1		)
				_lowerCamelCase     :			Optional[Any]    =    create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1		)

				with accelerator.join_uneven_inputs([ddp_model]							,    even_batches=lowercase__		):
								_lowerCamelCase     :			Dict    =    train_dl.batch_sampler.even_batches
								_lowerCamelCase     :			Union[str, Any]    =    valid_dl.batch_sampler.even_batches

				assert train_dl_overridden_value == overridden_even_batches
				assert valid_dl_overridden_value == overridden_even_batches
				assert train_dl.batch_sampler.even_batches == default_even_batches
				assert valid_dl.batch_sampler.even_batches == default_even_batches







def      _snake_case			(				):
				_lowerCamelCase     :			Dict    =    True
				_lowerCamelCase     :			Tuple    =    False
				_lowerCamelCase     :			Dict    =    create_accelerator(even_batches=lowercase__		)
				_lowerCamelCase     :			Dict    =    torch.nn.Linear(1							,    1		)
				_lowerCamelCase     :			Union[str, Any]    =    accelerator.prepare(lowercase__		)
				create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1							,    iterable=lowercase__		)
				_lowerCamelCase     :			str    =    create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1		)

				with warnings.catch_warnings():
								warnings.filterwarnings('ignore'		)
								try:
												with accelerator.join_uneven_inputs([ddp_model]							,    even_batches=lowercase__		):
																_lowerCamelCase     :			Any    =    batch_dl.batch_sampler.even_batches
								except AttributeError:
												# ensure attribute error is not raised when processing iterable dl
												raise AssertionError

				assert batch_dl_overridden_value == overridden_even_batches
				assert batch_dl.batch_sampler.even_batches == default_even_batches







def      _snake_case			(				):
				_lowerCamelCase     :			List[Any]    =    create_accelerator()
				_lowerCamelCase     :			Any    =    torch.nn.Linear(1							,    1		)
				_lowerCamelCase     :			Any    =    accelerator.prepare(lowercase__		)
				create_dataloader(lowercase__							,    dataset_size=3							,    batch_size=1							,    iterable=lowercase__		)

				with warnings.catch_warnings(record=lowercase__		) as w:
								with accelerator.join_uneven_inputs([ddp_model]							,    even_batches=lowercase__		):
												pass

								assert issubclass(w[-1].category							,    lowercase__		)
								assert "only supported for map-style datasets" in str(w[-1].message		)







def      _snake_case			(				):
				_lowerCamelCase     :			Optional[int]    =    create_accelerator()

				accelerator.print('Test that even_batches variable ensures uniform batches across processes'		)
				test_default_ensures_even_batch_sizes()

				accelerator.print('Run tests with even_batches disabled'		)
				test_can_disable_even_batches()

				accelerator.print('Test joining uneven inputs'		)
				test_can_join_uneven_inputs()

				accelerator.print('Test overriding even_batches when joining uneven inputs'		)
				test_join_can_override_even_batches()

				accelerator.print('Test overriding even_batches for mixed dataloader types'		)
				test_join_can_override_for_mixed_type_dataloaders()

				accelerator.print('Test overriding even_batches raises a warning for iterable dataloaders'		)
				test_join_raises_warning_for_iterable_when_overriding_even_batches()

				accelerator.print('Test join with non DDP distributed raises warning'		)
				_lowerCamelCase     :			Dict    =    accelerator.state.distributed_type
				_lowerCamelCase     :			List[Any]    =    DistributedType.FSDP
				test_join_raises_warning_for_non_ddp_distributed(lowercase__		)
				_lowerCamelCase     :			Union[str, Any]    =    original_state


if __name__ == "__main__":
			main()
"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_a      :		List[Any]           =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a      :		Union[str, Any]           =   {
    'vocab_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'
        ),
        'google/electra-base-generator': 'https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt',
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'google/electra-small-generator': (
            'https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-generator': (
            'https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-generator': (
            'https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'
        ),
        'google/electra-small-discriminator': (
            'https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-base-discriminator': (
            'https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'
        ),
        'google/electra-large-discriminator': (
            'https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'
        ),
    },
}

_a      :		Optional[Any]           =   {
    'google/electra-small-generator': 512,
    'google/electra-base-generator': 512,
    'google/electra-large-generator': 512,
    'google/electra-small-discriminator': 512,
    'google/electra-base-discriminator': 512,
    'google/electra-large-discriminator': 512,
}

_a      :		Any           =   {
    'google/electra-small-generator': {'do_lower_case': True},
    'google/electra-base-generator': {'do_lower_case': True},
    'google/electra-large-generator': {'do_lower_case': True},
    'google/electra-small-discriminator': {'do_lower_case': True},
    'google/electra-base-discriminator': {'do_lower_case': True},
    'google/electra-large-discriminator': {'do_lower_case': True},
}
class 						__A							( SCREAMING_SNAKE_CASE_		):
     _UpperCamelCase							:			Tuple							=		VOCAB_FILES_NAMES
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:			List[Any]							=		PRETRAINED_INIT_CONFIGURATION
     _UpperCamelCase							:			Union[str, Any]							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:			Optional[Any]							=		ElectraTokenizer






     def __init__(		self				,     a__=None				,     a__=None				,     a__=True				,     a__="[UNK]"				,     a__="[SEP]"				,     a__="[PAD]"				,     a__="[CLS]"				,     a__="[MASK]"				,     a__=True				,     a__=None				,     **a__				,     ):
       super().__init__(
           a__				,     tokenizer_file=a__				,     do_lower_case=a__				,     unk_token=a__				,     sep_token=a__				,     pad_token=a__				,     cls_token=a__				,     mask_token=a__				,     tokenize_chinese_chars=a__				,     strip_accents=a__				,     **a__				,     )

       _lowerCAmelCase						:       int  			=     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get("""lowercase"""				,     a__  ) != do_lower_case
           or normalizer_state.get("""strip_accents"""				,     a__  ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""				,     a__  ) != tokenize_chinese_chars
       ):
         _lowerCAmelCase						:       Dict  			=     getattr(a__				,     normalizer_state.pop("""type"""  )  )
         _lowerCAmelCase						:       int  			=     do_lower_case
         _lowerCAmelCase						:       str  			=     strip_accents
         _lowerCAmelCase						:       Dict  			=     tokenize_chinese_chars
         _lowerCAmelCase						:       str  			=     normalizer_class(**a__  )

       _lowerCAmelCase						:       List[str]  			=     do_lower_case






     def     __A  (		self				,     a__				,     a__=None  ):
       _lowerCAmelCase						:       int  			=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       List[str]  			=     [self.sep_token_id]
       _lowerCAmelCase						:       Any  			=     [self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






     def     __A  (		self				,     a__				,     a__ = None  ):
       _lowerCAmelCase						:       Optional[Any]  			=     self._tokenizer.model.save(a__				,     name=a__  )
       return tuple(a__  )




