



'''simple docstring'''



import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			_SCREAMING_SNAKE_CASE				(lowercase__,	unittest.TestCase      ):
			A__								=  HerbertTokenizer
			A__								=  HerbertTokenizerFast
			A__								=  True



			def 				lowerCAmelCase (     self		:						List[Any]			)      ->      List[str]:
							"""simple docstring"""





							super().setUp()

							# Use a simpler test file without japanese/chinese characters
							with open(F'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''    ,			encoding='''utf-8'''			) as f_data:
											snake_case__      :   List[Any]      =    f_data.read().replace('''\n\n'''    ,			'''\n'''			).strip()

							snake_case__      :   Any      =    [
							    '''<s>''',
							    '''</s>''',
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''w</w>''',
							    '''r</w>''',
							    '''t</w>''',
							    '''lo''',
							    '''low''',
							    '''er</w>''',
							    '''low</w>''',
							    '''lowest</w>''',
							    '''newer</w>''',
							    '''wider</w>''',
							    ''',</w>''',
							    '''<unk>''',
							]
							snake_case__      :   List[str]      =    dict(zip(__UpperCamelCase    ,			range(len(__UpperCamelCase			)			)			)			)
							snake_case__      :   List[str]      =    ['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

							snake_case__      :   int      =    os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['''vocab_file''']			)
							snake_case__      :   Dict      =    os.path.join(self.tmpdirname    ,			VOCAB_FILES_NAMES['''merges_file''']			)
							with open(self.vocab_file    ,			'''w'''			) as fp:
											fp.write(json.dumps(__UpperCamelCase			)			)
							with open(self.merges_file    ,			'''w'''			) as fp:
											fp.write('''\n'''.join(__UpperCamelCase			)			)



			def 				lowerCAmelCase (     self		:						int    ,			__UpperCamelCase		:						Dict			)      ->      Tuple:
							"""simple docstring"""





							snake_case__      :   Union[str, Any]      =    '''lower newer'''
							snake_case__      :   Optional[int]      =    '''lower newer'''
							return input_text, output_text



			def 				lowerCAmelCase (     self		:						Any			)      ->      List[Any]:
							"""simple docstring"""





							snake_case__      :   int      =    self.tokenizer_class(vocab_file=self.vocab_file    ,			merges_file=self.merges_file			)

							snake_case__      :   List[str]      =    '''lower'''
							snake_case__      :   Optional[Any]      =    ['''low''', '''er</w>''']
							snake_case__      :   Any      =    tokenizer.tokenize(__UpperCamelCase			)
							self.assertListEqual(__UpperCamelCase    ,			__UpperCamelCase			)

							snake_case__      :   str      =    tokens + ['''<unk>''']
							snake_case__      :   List[str]      =    [16, 17, 23]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase			)    ,			__UpperCamelCase			)



			def 				lowerCAmelCase (     self		:						List[str]			)      ->      str:
							"""simple docstring"""





							if not self.test_rust_tokenizer:
											return

							snake_case__      :   Optional[int]      =    self.get_tokenizer()
							snake_case__      :   Union[str, Any]      =    self.get_rust_tokenizer()

							snake_case__      :   Any      =    '''lower,newer'''

							snake_case__      :   Any      =    tokenizer.tokenize(__UpperCamelCase			)
							snake_case__      :   Union[str, Any]      =    rust_tokenizer.tokenize(__UpperCamelCase			)
							self.assertListEqual(__UpperCamelCase    ,			__UpperCamelCase			)

							snake_case__      :   Tuple      =    tokenizer.encode(__UpperCamelCase    ,			add_special_tokens=__UpperCamelCase			)
							snake_case__      :   Any      =    rust_tokenizer.encode(__UpperCamelCase    ,			add_special_tokens=__UpperCamelCase			)
							self.assertListEqual(__UpperCamelCase    ,			__UpperCamelCase			)

							snake_case__      :   Tuple      =    self.get_rust_tokenizer()
							snake_case__      :   Optional[Any]      =    tokenizer.encode(__UpperCamelCase			)
							snake_case__      :   Optional[int]      =    rust_tokenizer.encode(__UpperCamelCase			)
							self.assertListEqual(__UpperCamelCase    ,			__UpperCamelCase			)







			@slow
			def 				lowerCAmelCase (     self		:						str			)      ->      Union[str, Any]:
							"""simple docstring"""





							snake_case__      :   Optional[int]      =    self.tokenizer_class.from_pretrained('''allegro/herbert-base-cased'''			)

							snake_case__      :   Optional[int]      =    tokenizer.encode('''konstruowanie sekwencji'''    ,			add_special_tokens=__UpperCamelCase			)
							snake_case__      :   Dict      =    tokenizer.encode('''konstruowanie wielu sekwencji'''    ,			add_special_tokens=__UpperCamelCase			)

							snake_case__      :   int      =    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase			)
							snake_case__      :   str      =    tokenizer.build_inputs_with_special_tokens(__UpperCamelCase    ,			__UpperCamelCase			)

							assert encoded_sentence == [0] + text + [2]
							assert encoded_pair == [0] + text + [2] + text_a + [2]









import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_		:  str										=  logging.get_logger(__name__)

A_		:  Dict										=  {
    '''facebook/hubert-base-ls960''': '''https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json''',
    # See all Hubert models at https://huggingface.co/models?filter=hubert
}



class 		_lowercase		(							UpperCAmelCase__						):
   _UpperCAmelCase  						=      '''hubert'''

   def __init__(   self      :							Optional[Any]				, __lowerCAmelCase      :							str=32				, __lowerCAmelCase      :							int=768				, __lowerCAmelCase      :							Optional[int]=12				, __lowerCAmelCase      :							List[Any]=12				, __lowerCAmelCase      :							Optional[Any]=3072				, __lowerCAmelCase      :							Tuple="gelu"				, __lowerCAmelCase      :							Dict=0.1				, __lowerCAmelCase      :							List[Any]=0.1				, __lowerCAmelCase      :							str=0.1				, __lowerCAmelCase      :							Union[str, Any]=True				, __lowerCAmelCase      :							Tuple=0.0				, __lowerCAmelCase      :							Optional[int]=0.1				, __lowerCAmelCase      :							List[str]=0.1				, __lowerCAmelCase      :							Dict=0.0_2				, __lowerCAmelCase      :							Any=1E-5				, __lowerCAmelCase      :							Optional[Any]="group"				, __lowerCAmelCase      :							int="gelu"				, __lowerCAmelCase      :							Dict=(512, 512, 512, 512, 512, 512, 512)				, __lowerCAmelCase      :							Tuple=(5, 2, 2, 2, 2, 2, 2)				, __lowerCAmelCase      :							Tuple=(10, 3, 3, 3, 3, 2, 2)				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							Any=128				, __lowerCAmelCase      :							Tuple=16				, __lowerCAmelCase      :							Tuple=False				, __lowerCAmelCase      :							int=True				, __lowerCAmelCase      :							List[str]=0.0_5				, __lowerCAmelCase      :							List[str]=10				, __lowerCAmelCase      :							int=2				, __lowerCAmelCase      :							List[Any]=0.0				, __lowerCAmelCase      :							Dict=10				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							Union[str, Any]="sum"				, __lowerCAmelCase      :							Dict=False				, __lowerCAmelCase      :							List[str]=False				, __lowerCAmelCase      :							int=256				, __lowerCAmelCase      :							List[str]=0				, __lowerCAmelCase      :							str=1				, __lowerCAmelCase      :							Any=2				, **__lowerCAmelCase      :							Tuple				, )							->					Tuple:


    """simple docstring"""






    super().__init__(**__lowerCAmelCase				, pad_token_id=__lowerCAmelCase				, bos_token_id=__lowerCAmelCase				, eos_token_id=__lowerCAmelCase				)
    a							=		hidden_size
    a							=		feat_extract_norm
    a							=		feat_extract_activation
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		list(__lowerCAmelCase				)
    a							=		conv_bias
    a							=		num_conv_pos_embeddings
    a							=		num_conv_pos_embedding_groups
    a							=		len(self.conv_dim				)
    a							=		num_hidden_layers
    a							=		intermediate_size
    a							=		hidden_act
    a							=		num_attention_heads
    a							=		hidden_dropout
    a							=		attention_dropout
    a							=		activation_dropout
    a							=		feat_proj_layer_norm
    a							=		feat_proj_dropout
    a							=		final_dropout
    a							=		layerdrop
    a							=		layer_norm_eps
    a							=		initializer_range
    a							=		vocab_size
    a							=		do_stable_layer_norm
    a							=		use_weighted_layer_sum
    a							=		classifier_proj_size

    if (
        (len(self.conv_stride				) != self.num_feat_extract_layers)
        or (len(self.conv_kernel				) != self.num_feat_extract_layers)
        or (len(self.conv_dim				) != self.num_feat_extract_layers)
    ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         f""" {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,"""
         f""" `len(config.conv_kernel) = {len(self.conv_kernel				)}`."""				)

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    a							=		apply_spec_augment
    a							=		mask_time_prob
    a							=		mask_time_length
    a							=		mask_time_min_masks
    a							=		mask_feature_prob
    a							=		mask_feature_length
    a							=		mask_feature_min_masks

    # ctc loss
    a							=		ctc_loss_reduction
    a							=		ctc_zero_infinity




   @property
   def 		A				(   self      :							Tuple				)							->					Optional[Any]:


    """simple docstring"""






    return functools.reduce(operator.mul				, self.conv_stride				, 1				)

