






"""simple docstring"""
import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 						a_ (      snake_case_    ,		unittest.TestCase					):







      '''simple docstring'''

      lowerCamelCase__			:							int					=      HerbertTokenizer
      lowerCamelCase__			:							Union[str, Any]					=      HerbertTokenizerFast
      lowerCamelCase__			:							List[Any]					=      True




      def      a__				(self   ):

       '''simple docstring'''



       super().setUp()

       # Use a simpler test file without japanese/chinese characters
       with open(f'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt''',	encoding='utf-8'   ) as f_data:
        lowerCamelCase__	:     str								=		f_data.read().replace('\n\n',	'\n'   ).strip()

       lowerCamelCase__	:     Optional[int]								=		[
           '<s>',
           '</s>',
           'l',
           'o',
           'w',
           'e',
           'r',
           's',
           't',
           'i',
           'd',
           'n',
           'w</w>',
           'r</w>',
           't</w>',
           'lo',
           'low',
           'er</w>',
           'low</w>',
           'lowest</w>',
           'newer</w>',
           'wider</w>',
           ',</w>',
           '<unk>',
       ]
       lowerCamelCase__	:     Union[str, Any]								=		dict(zip(lowerCamelCase_,	range(len(lowerCamelCase_   )   )   )   )
       lowerCamelCase__	:     int								=		['l o 123', 'lo w 1456', 'e r</w> 1789', '']

       lowerCamelCase__	:     List[Any]								=		os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['vocab_file']   )
       lowerCamelCase__	:     str								=		os.path.join(self.tmpdirname,	VOCAB_FILES_NAMES['merges_file']   )
       with open(self.vocab_file,	'w'   ) as fp:
        fp.write(json.dumps(lowerCamelCase_   )   )
       with open(self.merges_file,	'w'   ) as fp:
        fp.write('\n'.join(lowerCamelCase_   )   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		'lower newer'
       lowerCamelCase__	:     Optional[Any]								=		'lower newer'
       return input_text, output_text




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[int]								=		self.tokenizer_class(vocab_file=self.vocab_file,	merges_file=self.merges_file   )

       lowerCamelCase__	:     Dict								=		'lower'
       lowerCamelCase__	:     str								=		['low', 'er</w>']
       lowerCamelCase__	:     Union[str, Any]								=		tokenizer.tokenize(lowerCamelCase_   )
       self.assertListEqual(lowerCamelCase_,	lowerCamelCase_   )

       lowerCamelCase__	:     List[Any]								=		tokens + ['<unk>']
       lowerCamelCase__	:     Dict								=		[1_6, 1_7, 2_3]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_   ),	lowerCamelCase_   )




      def      a__				(self   ):

       '''simple docstring'''



       if not self.test_rust_tokenizer:
        return

       lowerCamelCase__	:     int								=		self.get_tokenizer()
       lowerCamelCase__	:     str								=		self.get_rust_tokenizer()

       lowerCamelCase__	:     str								=		'lower,newer'

       lowerCamelCase__	:     Tuple								=		tokenizer.tokenize(lowerCamelCase_   )
       lowerCamelCase__	:     Tuple								=		rust_tokenizer.tokenize(lowerCamelCase_   )
       self.assertListEqual(lowerCamelCase_,	lowerCamelCase_   )

       lowerCamelCase__	:     str								=		tokenizer.encode(lowerCamelCase_,	add_special_tokens=lowerCamelCase_   )
       lowerCamelCase__	:     int								=		rust_tokenizer.encode(lowerCamelCase_,	add_special_tokens=lowerCamelCase_   )
       self.assertListEqual(lowerCamelCase_,	lowerCamelCase_   )

       lowerCamelCase__	:     Dict								=		self.get_rust_tokenizer()
       lowerCamelCase__	:     Tuple								=		tokenizer.encode(lowerCamelCase_   )
       lowerCamelCase__	:     Dict								=		rust_tokenizer.encode(lowerCamelCase_   )
       self.assertListEqual(lowerCamelCase_,	lowerCamelCase_   )





      @slow
      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Any								=		self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'   )

       lowerCamelCase__	:     Tuple								=		tokenizer.encode('konstruowanie sekwencji',	add_special_tokens=lowerCamelCase_   )
       lowerCamelCase__	:     Optional[int]								=		tokenizer.encode('konstruowanie wielu sekwencji',	add_special_tokens=lowerCamelCase_   )

       lowerCamelCase__	:     List[Any]								=		tokenizer.build_inputs_with_special_tokens(lowerCamelCase_   )
       lowerCamelCase__	:     List[Any]								=		tokenizer.build_inputs_with_special_tokens(lowerCamelCase_,	lowerCamelCase_   )

       assert encoded_sentence == [0] + text + [2]
       assert encoded_pair == [0] + text + [2] + text_a + [2]









"""simple docstring"""
from __future__ import annotations






def 			lowerCamelCase_    (		_lowerCamelCase							):
 lowerCamelCase__	:     int
 lowerCamelCase__	:     list[str]								=		list(_lowerCamelCase							)
 lowerCamelCase__							,	lowerCamelCase__	:     Union[str, Any]								=		_vector[:-1], int(_vector[-1]							)
 lowerCamelCase__	:     list[int]								=		[int(_lowerCamelCase							) for digit in __vector]

 vector.reverse()
 for i, digit in enumerate(_lowerCamelCase							):
  if i & 1 == 0:
   lowerCamelCase__	:     int								=		digit * 2
   if doubled > 9:
    doubled -= 9
   check_digit += doubled
  else:
   check_digit += digit

 return check_digit % 10 == 0


if __name__ == "__main__":
      import doctest

      doctest.testmod()
      assert is_luhn("79927398713")
      assert not is_luhn("79927398714")

