



"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase_   						=   logging.get_logger(__name__)

lowerCAmelCase_   						=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_   						=   [
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowerCAmelCase_   						=   {
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowerCAmelCase_   						=   {F'''funnel-transformer/{name}''': 512 for name in _model_names}
lowerCAmelCase_   						=   {F'''funnel-transformer/{name}''': {'do_lower_case': True} for name in _model_names}
class        __A       (	A_    ):







		'''simple docstring'''





		lowerCAmelCase			:					str   	=				VOCAB_FILES_NAMES
		lowerCAmelCase			:					List[str]   	=				PRETRAINED_VOCAB_FILES_MAP
		lowerCAmelCase			:					int   	=				PRETRAINED_INIT_CONFIGURATION
		lowerCAmelCase			:					Dict   	=				FunnelTokenizer
		lowerCAmelCase			:					Tuple   	=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		lowerCAmelCase			:					int   	=				2







		def __init__(	self						:  Optional[int]			,_snake_case						:  Dict=None			,_snake_case						:  List[str]=None			,_snake_case						:  Optional[Any]=True			,_snake_case						:  Optional[int]="<unk>"			,_snake_case						:  Dict="<sep>"			,_snake_case						:  Any="<pad>"			,_snake_case						:  str="<cls>"			,_snake_case						:  Optional[Any]="<mask>"			,_snake_case						:  int="<s>"			,_snake_case						:  Dict="</s>"			,_snake_case						:  Optional[int]=True			,_snake_case						:  List[str]=True			,_snake_case						:  Dict=None			,_snake_case						:  str="##"			,**_snake_case						:  Optional[Any]			,)					->		Any:






				"""simple docstring"""



				super().__init__(
				    _snake_case			,tokenizer_file=_snake_case			,do_lower_case=_snake_case			,unk_token=_snake_case			,sep_token=_snake_case			,pad_token=_snake_case			,cls_token=_snake_case			,mask_token=_snake_case			,bos_token=_snake_case			,eos_token=_snake_case			,clean_text=_snake_case			,tokenize_chinese_chars=_snake_case			,strip_accents=_snake_case			,wordpieces_prefix=_snake_case			,**_snake_case			,)

				lowercase__   :    Union[str, Any]				=			json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
				if (
				    normalizer_state.get('''lowercase'''			,_snake_case						) != do_lower_case
				    or normalizer_state.get('''strip_accents'''			,_snake_case						) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''			,_snake_case						) != tokenize_chinese_chars
				):
						lowercase__   :    List[str]				=			getattr(_snake_case			,normalizer_state.pop('''type'''						)						)
						lowercase__   :    List[str]				=			do_lower_case
						lowercase__   :    Any				=			strip_accents
						lowercase__   :    Union[str, Any]				=			tokenize_chinese_chars
						lowercase__   :    Union[str, Any]				=			normalizer_class(**_snake_case						)

				lowercase__   :    List[str]				=			do_lower_case







		def 					UpperCAmelCase    (	self						:  int			,_snake_case						:  Tuple			,_snake_case						:  Tuple=None						)					->		Optional[Any]:






				"""simple docstring"""



				lowercase__   :    Any				=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

				return output







		def 					UpperCAmelCase    (	self						:  Tuple			,_snake_case						:  List[int]			,_snake_case						:  Optional[List[int]] = None						)					->		List[int]:






				"""simple docstring"""



				lowercase__   :    Optional[int]				=			[self.sep_token_id]
				lowercase__   :    List[Any]				=			[self.cls_token_id]
				if token_ids_a is None:
						return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
				return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







		def 					UpperCAmelCase    (	self						:  Tuple			,_snake_case						:  str			,_snake_case						:  Optional[str] = None						)					->		Tuple[str]:






				"""simple docstring"""



				lowercase__   :    str				=			self._tokenizer.model.save(_snake_case			,name=_snake_case						)
				return tuple(_snake_case						)





'''simple docstring'''


import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_lowerCamelCase	:       List[str]      	=	5_0000
_lowerCamelCase	:       Optional[int]      	=	5000

_lowerCamelCase	,_lowerCamelCase	:       int      	=	os.path.split(__file__)
_lowerCamelCase	:       str      	=	os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))







@get_duration
def 				__lowerCamelCase				(   A__    ,						A__    )       ->	Any:






   """simple docstring"""
   for i in range(A__    ):
      UpperCamelCase					=  dataset[i]







@get_duration
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    )       ->	int:






   """simple docstring"""
   for i in range(0    ,						len(A__    )    ,						A__    ):
      UpperCamelCase					=  dataset[i : i + batch_size]







@get_duration
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    )       ->	List[Any]:






   """simple docstring"""
   with dataset.formatted_as(type=A__    ):
      for i in range(A__    ):
         UpperCamelCase					=  dataset[i]







@get_duration
def 				__lowerCamelCase				(   A__    ,						A__    ,						A__    ,						A__    )       ->	int:






   """simple docstring"""
   with dataset.formatted_as(type=A__    ):
      for i in range(0    ,						A__    ,						A__    ):
         UpperCamelCase					=  dataset[i : i + batch_size]







def 				__lowerCamelCase				(   )       ->	List[str]:






   """simple docstring"""
   UpperCamelCase					=  {'num examples': SPEED_TEST_N_EXAMPLES}
   UpperCamelCase					=  [
       (read, {'length': SMALL_TEST}),
       (read, {'length': SPEED_TEST_N_EXAMPLES}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_000}),
       (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_000}),
   ]

   UpperCamelCase					=  [
       (read, {'length': SMALL_TEST}),
       (read, {'length': SPEED_TEST_N_EXAMPLES}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_000}),
       (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_000}),
   ]
   with tempfile.TemporaryDirectory() as tmp_dir:
      print('generating dataset'    )
      UpperCamelCase					=  datasets.Features(
          {'list': datasets.Sequence(datasets.Value('float32'    )    ), 'numbers': datasets.Value('float32'    )}    )
      UpperCamelCase					=  generate_example_dataset(
          os.path.join(A__    ,						'dataset.arrow'    )    ,						A__    ,						num_examples=A__    ,						seq_shapes={'list': (100,)}    ,						)
      print('first set of iterations'    )
      for func, kwargs in functions:
         print(func.__name__    ,						str(A__    )    )
         UpperCamelCase					=  func(A__    ,						**A__    )

      print('shuffling dataset'    )
      UpperCamelCase					=  dataset.shuffle()
      print('Second set of iterations (after shuffling'    )
      for func, kwargs in functions_shuffled:
         print('shuffled '    ,						func.__name__    ,						str(A__    )    )
         UpperCamelCase					=  func(
             A__    ,						**A__    )

   with open(A__    ,						'wb'    ) as f:
      f.write(json.dumps(A__    ).encode('utf-8'    )    )


if __name__ == "__main__":  # useful to run the profiler
   benchmark_iterating()

