



'''simple docstring'''





import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_						, unittest.TestCase     ):
  '''simple docstring'''




  __lowercase							:      Optional[int]				=      LongformerTokenizer
  __lowercase							:      Union[str, Any]				=      True
  __lowercase							:      Any				=      LongformerTokenizerFast
  __lowercase							:      int				=      True




  def 							UpperCAmelCase_	(    self      )    ->		List[str]:
      super().setUp()

      # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
      lowerCAmelCase__ :  str							      =					[
          """l""",
          """o""",
          """w""",
          """e""",
          """r""",
          """s""",
          """t""",
          """i""",
          """d""",
          """n""",
          """\u0120""",
          """\u0120l""",
          """\u0120n""",
          """\u0120lo""",
          """\u0120low""",
          """er""",
          """\u0120lowest""",
          """\u0120newer""",
          """\u0120wider""",
          """<unk>""",
      ]
      lowerCAmelCase__ :  Tuple							      =					dict(zip(__UpperCAmelCase  ,range(len(__UpperCAmelCase      )      )      )      )
      lowerCAmelCase__ :  int							      =					["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
      lowerCAmelCase__ :  Union[str, Any]							      =					{"""unk_token""": """<unk>"""}

      lowerCAmelCase__ :  List[Any]							      =					os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]      )
      lowerCAmelCase__ :  int							      =					os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""merges_file"""]      )
      with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""      ) as fp:
          fp.write(json.dumps(__UpperCAmelCase      ) + """\n"""      )
      with open(self.merges_file  ,"""w"""  ,encoding="""utf-8"""      ) as fp:
          fp.write("""\n""".join(__UpperCAmelCase      )      )




  def 							UpperCAmelCase_	(    self  ,**__UpperCAmelCase      )    ->		Union[str, Any]:
      kwargs.update(self.special_tokens_map      )
      return self.tokenizer_class.from_pretrained(self.tmpdirname  ,**__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,**__UpperCAmelCase      )    ->		List[str]:
      kwargs.update(self.special_tokens_map      )
      return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,**__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Tuple:
      lowerCAmelCase__ :  Union[str, Any]							      =					"""lower newer"""
      lowerCAmelCase__ :  Union[str, Any]							      =					"""lower newer"""
      return input_text, output_text




  def 							UpperCAmelCase_	(    self      )    ->		Any:
      lowerCAmelCase__ :  Optional[Any]							      =					self.tokenizer_class(self.vocab_file  ,self.merges_file  ,**self.special_tokens_map      )
      lowerCAmelCase__ :  Optional[Any]							      =					"""lower newer"""
      lowerCAmelCase__ :  List[Any]							      =					["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
      lowerCAmelCase__ :  Any							      =					tokenizer.tokenize(__UpperCAmelCase      )  # , add_prefix_space=True)
      self.assertListEqual(__UpperCAmelCase  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  List[str]							      =					tokens + [tokenizer.unk_token]
      lowerCAmelCase__ :  int							      =					[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase      )  ,__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		List[Any]:
      lowerCAmelCase__ :  Optional[Any]							      =					self.get_tokenizer()

      self.assertListEqual(tokenizer.encode("""Hello world!"""  ,add_special_tokens=__UpperCAmelCase      )  ,[0, 3_1414, 232, 328, 2]      )
      self.assertListEqual(
          tokenizer.encode("""Hello world! cécé herlolip 418"""  ,add_special_tokens=__UpperCAmelCase      )  ,[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]  ,)




  @slow
  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      lowerCAmelCase__ :  Optional[Any]							      =					self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""      )

      lowerCAmelCase__ :  List[str]							      =					tokenizer.encode("""sequence builders"""  ,add_special_tokens=__UpperCAmelCase      )
      lowerCAmelCase__ :  Any							      =					tokenizer.encode("""multi-sequence build"""  ,add_special_tokens=__UpperCAmelCase      )

      lowerCAmelCase__ :  int							      =					tokenizer.encode(
          """sequence builders"""  ,add_special_tokens=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[Any]							      =					tokenizer.encode(
          """sequence builders"""  ,"""multi-sequence build"""  ,add_special_tokens=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[Any]							      =					tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase  ,__UpperCAmelCase      )

      assert encoded_sentence == encoded_text_from_decode
      assert encoded_pair == encoded_pair_from_decode




  def 							UpperCAmelCase_	(    self      )    ->		Dict:
      lowerCAmelCase__ :  Union[str, Any]							      =					self.get_tokenizer()

      lowerCAmelCase__ :  List[str]							      =					"""Encode this sequence."""
      lowerCAmelCase__ :  Tuple							      =					tokenizer.byte_encoder[""" """.encode("""utf-8"""      )[0]]

      # Testing encoder arguments
      lowerCAmelCase__ :  str							      =					tokenizer.encode(__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					tokenizer.convert_ids_to_tokens(encoded[0]      )[0]
      self.assertNotEqual(__UpperCAmelCase  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					tokenizer.encode(__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					tokenizer.convert_ids_to_tokens(encoded[0]      )[0]
      self.assertEqual(__UpperCAmelCase  ,__UpperCAmelCase      )

      tokenizer.add_special_tokens({"""bos_token""": """<s>"""}      )
      lowerCAmelCase__ :  int							      =					tokenizer.encode(__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
      lowerCAmelCase__ :  List[str]							      =					tokenizer.convert_ids_to_tokens(encoded[1]      )[0]
      self.assertNotEqual(__UpperCAmelCase  ,__UpperCAmelCase      )

      # Testing spaces after special tokens
      lowerCAmelCase__ :  Optional[int]							      =					"""<mask>"""
      tokenizer.add_special_tokens(
          {"""mask_token""": AddedToken(__UpperCAmelCase  ,lstrip=__UpperCAmelCase  ,rstrip=__UpperCAmelCase      )}      )  # mask token has a left space
      lowerCAmelCase__ :  Optional[int]							      =					tokenizer.convert_tokens_to_ids(__UpperCAmelCase      )

      lowerCAmelCase__ :  Optional[int]							      =					"""Encode <mask> sequence"""
      lowerCAmelCase__ :  int							      =					"""Encode <mask>sequence"""

      lowerCAmelCase__ :  str							      =					tokenizer.encode(__UpperCAmelCase      )
      lowerCAmelCase__ :  int							      =					encoded.index(__UpperCAmelCase      )
      lowerCAmelCase__ :  Any							      =					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]      )[0]
      self.assertEqual(__UpperCAmelCase  ,__UpperCAmelCase      )

      lowerCAmelCase__ :  Union[str, Any]							      =					tokenizer.encode(__UpperCAmelCase      )
      lowerCAmelCase__ :  str							      =					encoded.index(__UpperCAmelCase      )
      lowerCAmelCase__ :  List[str]							      =					tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]      )[0]
      self.assertNotEqual(__UpperCAmelCase  ,__UpperCAmelCase      )




  def 							UpperCAmelCase_	(    self      )    ->		List[str]:
      pass




  def 							UpperCAmelCase_	(    self      )    ->		Any:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
              lowerCAmelCase__ :  Union[str, Any]							      =					self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase  ,**__UpperCAmelCase      )
              lowerCAmelCase__ :  Dict							      =					self.tokenizer_class.from_pretrained(__UpperCAmelCase  ,**__UpperCAmelCase      )
              lowerCAmelCase__ :  Optional[Any]							      =					"""A, <mask> AllenNLP sentence."""
              lowerCAmelCase__ :  int							      =					tokenizer_r.encode_plus(__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase  ,return_token_type_ids=__UpperCAmelCase      )
              lowerCAmelCase__ :  Union[str, Any]							      =					tokenizer_p.encode_plus(__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase  ,return_token_type_ids=__UpperCAmelCase      )

              # token_type_ids should put 0 everywhere
              self.assertEqual(sum(tokens_r["""token_type_ids"""]      )  ,sum(tokens_p["""token_type_ids"""]      )      )

              # attention_mask should put 1 everywhere, so sum over length should be 1
              self.assertEqual(
                  sum(tokens_r["""attention_mask"""]      ) / len(tokens_r["""attention_mask"""]      )  ,sum(tokens_p["""attention_mask"""]      ) / len(tokens_p["""attention_mask"""]      )  ,)

              lowerCAmelCase__ :  Union[str, Any]							      =					tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]      )
              lowerCAmelCase__ :  List[str]							      =					tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]      )

              # Rust correctly handles the space before the mask while python doesnt
              self.assertSequenceEqual(tokens_p["""input_ids"""]  ,[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]      )
              self.assertSequenceEqual(tokens_r["""input_ids"""]  ,[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]      )

              self.assertSequenceEqual(
                  __UpperCAmelCase  ,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]      )
              self.assertSequenceEqual(
                  __UpperCAmelCase  ,["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]      )




  def 							UpperCAmelCase_	(    self      )    ->		int:
      for trim_offsets, add_prefix_space in itertools.product([True, False]  ,repeat=2      ):
          lowerCAmelCase__ :  Tuple							      =					self.rust_tokenizer_class.from_pretrained(
              self.tmpdirname  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )

          lowerCAmelCase__ :  int							      =					json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()      )
          lowerCAmelCase__ :  List[str]							      =					json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()      )

          self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]  ,__UpperCAmelCase      )

          self.assertEqual(post_processor_state["""add_prefix_space"""]  ,__UpperCAmelCase      )
          self.assertEqual(post_processor_state["""trim_offsets"""]  ,__UpperCAmelCase      )






  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
      # `trim_offsets`
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
              lowerCAmelCase__ :  Dict							      =					"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
              lowerCAmelCase__ :  Any							      =					F"""{text_of_1_token} {text_of_1_token}"""

              lowerCAmelCase__ :  Tuple							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  List[str]							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(len(__UpperCAmelCase      ) + 1, len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  Optional[int]							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  Any							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(len(__UpperCAmelCase      ) + 1, len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  Union[str, Any]							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  Optional[Any]							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(len(__UpperCAmelCase      ), len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  Tuple							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  Tuple							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(len(__UpperCAmelCase      ), len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  int							      =					F""" {text}"""

              # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
              #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
              # )
              # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
              # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
              # self.assertEqual(
              #     encoding.offset_mapping[1],
              #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
              # )

              lowerCAmelCase__ :  Tuple							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  Any							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(1, 1 + len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(1 + len(__UpperCAmelCase      ) + 1, 1 + len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  Any							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  str							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, 1 + len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(1 + len(__UpperCAmelCase      ), 1 + len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)

              lowerCAmelCase__ :  Optional[Any]							      =					self.rust_tokenizer_class.from_pretrained(
                  __UpperCAmelCase  ,use_fast=__UpperCAmelCase  ,add_prefix_space=__UpperCAmelCase  ,trim_offsets=__UpperCAmelCase      )
              lowerCAmelCase__ :  List[Any]							      =					tokenizer_r(__UpperCAmelCase  ,return_offsets_mapping=__UpperCAmelCase  ,add_special_tokens=__UpperCAmelCase      )
              self.assertEqual(encoding.offset_mapping[0]  ,(0, 1 + len(__UpperCAmelCase      ))      )
              self.assertEqual(
                  encoding.offset_mapping[1]  ,(1 + len(__UpperCAmelCase      ), 1 + len(__UpperCAmelCase      ) + 1 + len(__UpperCAmelCase      ))  ,)







'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_lowerCAmelCase      						=	logging.get_logger(__name__)

_lowerCAmelCase      						=	{
    '''microsoft/focalnet-tiny''': '''https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json''',
}






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_						, SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      Optional[Any]				=      '''focalnet'''




  def __init__(    self  ,__UpperCAmelCase=224  ,__UpperCAmelCase=4  ,__UpperCAmelCase=3  ,__UpperCAmelCase=96  ,__UpperCAmelCase=False  ,__UpperCAmelCase=[192, 384, 768, 768]  ,__UpperCAmelCase=[2, 2, 6, 2]  ,__UpperCAmelCase=[2, 2, 2, 2]  ,__UpperCAmelCase=[3, 3, 3, 3]  ,__UpperCAmelCase="gelu"  ,__UpperCAmelCase=4.0  ,__UpperCAmelCase=0.0  ,__UpperCAmelCase=0.1  ,__UpperCAmelCase=False  ,__UpperCAmelCase=1E-4  ,__UpperCAmelCase=False  ,__UpperCAmelCase=False  ,__UpperCAmelCase=False  ,__UpperCAmelCase=0.0_2  ,__UpperCAmelCase=1E-5  ,__UpperCAmelCase=32  ,__UpperCAmelCase=None  ,__UpperCAmelCase=None  ,**__UpperCAmelCase  ,)    ->		Optional[Any]:
      super().__init__(**__UpperCAmelCase      )

      lowerCAmelCase__ :  Dict							      =					image_size
      lowerCAmelCase__ :  int							      =					patch_size
      lowerCAmelCase__ :  str							      =					num_channels
      lowerCAmelCase__ :  Dict							      =					embed_dim
      lowerCAmelCase__ :  List[str]							      =					use_conv_embed
      lowerCAmelCase__ :  List[Any]							      =					hidden_sizes
      lowerCAmelCase__ :  Dict							      =					depths
      lowerCAmelCase__ :  List[str]							      =					focal_levels
      lowerCAmelCase__ :  List[str]							      =					focal_windows
      lowerCAmelCase__ :  Dict							      =					hidden_act
      lowerCAmelCase__ :  Dict							      =					mlp_ratio
      lowerCAmelCase__ :  Tuple							      =					hidden_dropout_prob
      lowerCAmelCase__ :  Tuple							      =					drop_path_rate
      lowerCAmelCase__ :  Dict							      =					use_layerscale
      lowerCAmelCase__ :  Optional[Any]							      =					layerscale_value
      lowerCAmelCase__ :  str							      =					use_post_layernorm
      lowerCAmelCase__ :  Union[str, Any]							      =					use_post_layernorm_in_modulation
      lowerCAmelCase__ :  int							      =					normalize_modulator
      lowerCAmelCase__ :  Optional[Any]							      =					initializer_range
      lowerCAmelCase__ :  List[str]							      =					layer_norm_eps
      lowerCAmelCase__ :  List[Any]							      =					encoder_stride
      lowerCAmelCase__ :  Dict							      =					["""stem"""] + [F"""stage{idx}""" for idx in range(1  ,len(self.depths      ) + 1      )]
      lowerCAmelCase__      ,   lowerCAmelCase__ :  Any							      =					get_aligned_output_features_output_indices(
          out_features=__UpperCAmelCase  ,out_indices=__UpperCAmelCase  ,stage_names=self.stage_names      )


