

"""simple docstring"""




import argparse
import collections
import importlib.util
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
SCREAMING_SNAKE_CASE_  :      Tuple							=      '''src/diffusers'''
SCREAMING_SNAKE_CASE_  :      Dict							=      '''docs/source/en'''
SCREAMING_SNAKE_CASE_  :      List[Any]							=      '''.'''



def 						UpperCAmelCase__ (					A__						,     A__						,     A__ )   ->   List[str]:




   """simple docstring"""
   with open(A__						,     "r"						,     encoding="utf-8"						,     newline="\n" ) as f:
      lowerCamelCase__						       =      f.readlines()
   # Find the start prompt.
   lowerCamelCase__						       =      0
   while not lines[start_index].startswith(A__ ):
      start_index += 1
   start_index += 1

   lowerCamelCase__						       =      start_index
   while not lines[end_index].startswith(A__ ):
      end_index += 1
   end_index -= 1

   while len(lines[start_index] ) <= 1:
      start_index += 1
   while len(lines[end_index] ) <= 1:
      end_index -= 1
   end_index += 1
   return "".join(lines[start_index:end_index] ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
SCREAMING_SNAKE_CASE_  :      List[Any]							=      '''Model|Encoder|Decoder|ForConditionalGeneration'''
# Regexes that match TF/Flax/PT model names.
SCREAMING_SNAKE_CASE_  :      int							=      re.compile(r'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
SCREAMING_SNAKE_CASE_  :      Optional[int]							=      re.compile(r'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
SCREAMING_SNAKE_CASE_  :      Optional[int]							=      re.compile(r'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# This is to make sure the diffusers module imported is the one in the repo.
SCREAMING_SNAKE_CASE_  :      List[str]							=      importlib.util.spec_from_file_location(
    '''diffusers''',
    os.path.join(TRANSFORMERS_PATH, '''__init__.py'''),
    submodule_search_locations=[TRANSFORMERS_PATH],
)
SCREAMING_SNAKE_CASE_  :      List[Any]							=      spec.loader.load_module()



def 						UpperCAmelCase__ (					A__ )   ->   List[Any]:




   """simple docstring"""
   lowerCamelCase__						       =      re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"						,     A__ )
   return [m.group(0 ) for m in matches]



def 						UpperCAmelCase__ (					A__						,     A__ )   ->   str:




   """simple docstring"""
   lowerCamelCase__						       =      2 if text == "✅" or text == "❌" else len(A__ )
   lowerCamelCase__						       =      (width - text_length) // 2
   lowerCamelCase__						       =      width - text_length - left_indent
   return " " * left_indent + text + " " * right_indent



def 						UpperCAmelCase__ (					)   ->   List[Any]:




   """simple docstring"""
   lowerCamelCase__						       =      diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
   lowerCamelCase__						       =      {
       name: config_mapping_names[code]
       for code, name in diffusers_module.MODEL_NAMES_MAPPING.items()
       if code in config_mapping_names
   }
   lowerCamelCase__						       =      {name: config.replace("ConfigMixin"						,     "" ) for name, config in model_name_to_config.items()}

   # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
   lowerCamelCase__						       =      collections.defaultdict(A__ )
   lowerCamelCase__						       =      collections.defaultdict(A__ )
   lowerCamelCase__						       =      collections.defaultdict(A__ )
   lowerCamelCase__						       =      collections.defaultdict(A__ )
   lowerCamelCase__						       =      collections.defaultdict(A__ )

   # Let's lookup through all diffusers object (once).
   for attr_name in dir(A__ ):
      lowerCamelCase__						       =      None
      if attr_name.endswith("Tokenizer" ):
         lowerCamelCase__						       =      slow_tokenizers
         lowerCamelCase__						       =      attr_name[:-9]
      elif attr_name.endswith("TokenizerFast" ):
         lowerCamelCase__						       =      fast_tokenizers
         lowerCamelCase__						       =      attr_name[:-13]
      elif _re_tf_models.match(A__ ) is not None:
         lowerCamelCase__						       =      tf_models
         lowerCamelCase__						       =      _re_tf_models.match(A__ ).groups()[0]
      elif _re_flax_models.match(A__ ) is not None:
         lowerCamelCase__						       =      flax_models
         lowerCamelCase__						       =      _re_flax_models.match(A__ ).groups()[0]
      elif _re_pt_models.match(A__ ) is not None:
         lowerCamelCase__						       =      pt_models
         lowerCamelCase__						       =      _re_pt_models.match(A__ ).groups()[0]

      if lookup_dict is not None:
         while len(A__ ) > 0:
            if attr_name in model_name_to_prefix.values():
               lowerCamelCase__						       =      True
               break
            # Try again after removing the last word in the name
            lowerCamelCase__						       =      "".join(camel_case_split(A__ )[:-1] )

    # Let's build that table!
   lowerCamelCase__						       =      list(model_name_to_config.keys() )
   model_names.sort(key=str.lower )
   lowerCamelCase__						       =      ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
   # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
   lowerCamelCase__						       =      [len(A__ ) + 2 for c in columns]
   lowerCamelCase__						       =      max([len(A__ ) for name in model_names] ) + 2

   # Build the table per se
   lowerCamelCase__						       =      "|" + "|".join([_center_text(A__						,     A__ ) for c, w in zip(A__						,     A__ )] ) + "|\n"
   # Use ":-----:" format to center-aligned table cell texts
   table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths] ) + "|\n"

   lowerCamelCase__						       =      {True: "✅", False: "❌"}
   for name in model_names:
      lowerCamelCase__						       =      model_name_to_prefix[name]
      lowerCamelCase__						       =      [
          name,
          check[slow_tokenizers[prefix]],
          check[fast_tokenizers[prefix]],
          check[pt_models[prefix]],
          check[tf_models[prefix]],
          check[flax_models[prefix]],
      ]
      table += "|" + "|".join([_center_text(A__						,     A__ ) for l, w in zip(A__						,     A__ )] ) + "|\n"
   return table



def 						UpperCAmelCase__ (					A__=False )   ->   Dict:




   """simple docstring"""
   lowerCamelCase__       ,							lowerCamelCase__       ,							lowerCamelCase__       ,							lowerCamelCase__						       =      _find_text_in_file(
       filename=os.path.join(A__						,     "index.mdx" )						,     start_prompt="<!--This table is updated automatically from the auto modules"						,     end_prompt="<!-- End table-->"						,     )
   lowerCamelCase__						       =      get_model_table_from_auto_modules()

   if current_table != new_table:
      if overwrite:
         with open(os.path.join(A__						,     "index.mdx" )						,     "w"						,     encoding="utf-8"						,     newline="\n" ) as f:
            f.writelines(lines[:start_index] + [new_table] + lines[end_index:] )
      else:
         raise ValueError(
             "The model table in the `index.mdx` has not been updated. Run `make fix-copies` to fix this." )


if __name__ == "__main__":
     SCREAMING_SNAKE_CASE_  :      str							=      argparse.ArgumentParser()
     parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
     SCREAMING_SNAKE_CASE_  :      Any							=      parser.parse_args()

     check_model_table(args.fix_and_overwrite)







"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


SCREAMING_SNAKE_CASE_  :      str							=      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_  :      List[Any]							=      {'''vocab_file''': '''vocab.txt'''}

SCREAMING_SNAKE_CASE_  :      int							=      {
    '''vocab_file''': {
        '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt''',
        '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt''',
        '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt''',
        '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt''',
    }
}

SCREAMING_SNAKE_CASE_  :      Union[str, Any]							=      {
    '''tau/splinter-base''': 512,
    '''tau/splinter-base-qass''': 512,
    '''tau/splinter-large''': 512,
    '''tau/splinter-large-qass''': 512,
}

SCREAMING_SNAKE_CASE_  :      List[Any]							=      {
    '''tau/splinter-base''': {'''do_lower_case''': False},
    '''tau/splinter-base-qass''': {'''do_lower_case''': False},
    '''tau/splinter-large''': {'''do_lower_case''': False},
    '''tau/splinter-large-qass''': {'''do_lower_case''': False},
}


class 		_A							(      __a       ):
      __a             =						VOCAB_FILES_NAMES
      __a             =						PRETRAINED_VOCAB_FILES_MAP
      __a             =						PRETRAINED_INIT_CONFIGURATION
      __a             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a             =						SplinterTokenizer





      def __init__(	self     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=None     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__="[UNK]"     ,    SCREAMING_SNAKE_CASE__="[SEP]"     ,    SCREAMING_SNAKE_CASE__="[PAD]"     ,    SCREAMING_SNAKE_CASE__="[CLS]"     ,    SCREAMING_SNAKE_CASE__="[MASK]"     ,    SCREAMING_SNAKE_CASE__="[QUESTION]"     ,    SCREAMING_SNAKE_CASE__=True     ,    SCREAMING_SNAKE_CASE__=None     ,    **SCREAMING_SNAKE_CASE__     ,    ) ->       Optional[Any]:
         super().__init__(
             SCREAMING_SNAKE_CASE__     ,    tokenizer_file=SCREAMING_SNAKE_CASE__     ,    do_lower_case=SCREAMING_SNAKE_CASE__     ,    unk_token=SCREAMING_SNAKE_CASE__     ,    sep_token=SCREAMING_SNAKE_CASE__     ,    pad_token=SCREAMING_SNAKE_CASE__     ,    cls_token=SCREAMING_SNAKE_CASE__     ,    mask_token=SCREAMING_SNAKE_CASE__     ,    tokenize_chinese_chars=SCREAMING_SNAKE_CASE__     ,    strip_accents=SCREAMING_SNAKE_CASE__     ,    additional_special_tokens=(question_token,)     ,    **SCREAMING_SNAKE_CASE__     ,    )

         lowerCamelCase__						       =      json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             pre_tok_state.get("lowercase"     ,    SCREAMING_SNAKE_CASE__		) != do_lower_case
             or pre_tok_state.get("strip_accents"     ,    SCREAMING_SNAKE_CASE__		) != strip_accents
         ):
            lowerCamelCase__						       =      getattr(SCREAMING_SNAKE_CASE__     ,    pre_tok_state.pop("type"		)		)
            lowerCamelCase__						       =      do_lower_case
            lowerCamelCase__						       =      strip_accents
            lowerCamelCase__						       =      pre_tok_class(**SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						       =      do_lower_case





      @property
      def        _lowerCamelCase	(	self		) ->       Optional[Any]:
         return self.convert_tokens_to_ids(self.question_token		)





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       List[int]:
         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         lowerCamelCase__						       =      [self.cls_token_id]
         lowerCamelCase__						       =      [self.sep_token_id]
         lowerCamelCase__						       =      [self.question_token_id] + [self.convert_tokens_to_ids("."		)]
         if self.padding_side == "right":
            # Input is question-then-context
            return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
            # Input is context-then-question
            return cls + token_ids_a + sep + token_ids_a + question_suffix + sep





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       List[int]:
         lowerCamelCase__						       =      [self.sep_token_id]
         lowerCamelCase__						       =      [self.cls_token_id]
         lowerCamelCase__						       =      [self.question_token_id] + [self.convert_tokens_to_ids("."		)]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep		) * [0]

         if self.padding_side == "right":
            # Input is question-then-context
            return len(cls + token_ids_a + question_suffix + sep		) * [0] + len(token_ids_a + sep		) * [1]
         else:
            # Input is context-then-question
            return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + question_suffix + sep		) * [1]





      def        _lowerCamelCase	(	self     ,    SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__ = None		) ->       Tuple[str]:
         lowerCamelCase__						       =      self._tokenizer.model.save(SCREAMING_SNAKE_CASE__     ,    name=SCREAMING_SNAKE_CASE__		)
         return tuple(SCREAMING_SNAKE_CASE__		)




