
'''simple docstring'''





import tempfile
import unittest

import transformers
from transformers import XGLMConfig, XGLMTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
 import jax
 import jax.numpy as jnp
 import numpy as np

 from transformers.modeling_flax_pytorch_utils import (
     convert_pytorch_state_dict_to_flax,
     load_flax_weights_in_pytorch_model,
 )
 from transformers.models.xglm.modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel


if is_torch_available():
 import torch


@require_flax
class 			UpperCamelCase__	:






 def __init__(      self	:List[str]						,					_A	:int						,					_A	:Any=14						,					_A	:Union[str, Any]=7						,					_A	:Optional[int]=True						,					_A	:List[str]=True						,					_A	:int=True						,					_A	:Any=99						,					_A	:Optional[int]=32						,					_A	:List[str]=5						,					_A	:Optional[Any]=4						,					_A	:Union[str, Any]=37						,					_A	:List[Any]="gelu"						,					_A	:int=0.1						,					_A	:Union[str, Any]=0.1						,					_A	:str=512						,					_A	:Optional[Any]=0.02						,					_A	:Tuple=None						,					)					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      parent
       __A													=      batch_size
       __A													=      seq_length
       __A													=      is_training
       __A													=      use_input_mask
       __A													=      use_labels
       __A													=      vocab_size
       __A													=      d_model
       __A													=      num_hidden_layers
       __A													=      num_attention_heads
       __A													=      ffn_dim
       __A													=      activation_function
       __A													=      activation_dropout
       __A													=      attention_dropout
       __A													=      max_position_embeddings
       __A													=      initializer_range
       __A													=      None
       __A													=      0
       __A													=      2
       __A													=      1






 def 						lowercase_    (      self	:Union[str, Any]      )					->      Any:







       '''simple docstring'''



       __A													=      np.clip(ids_tensor([self.batch_size, self.seq_length]						,					self.vocab_size      )						,					3						,					self.vocab_size      )

       __A													=      None
       if self.use_input_mask:
             __A													=      random_attention_mask([self.batch_size, self.seq_length]      )

       __A													=      XGLMConfig(
           vocab_size=self.vocab_size						,					d_model=self.hidden_size						,					num_layers=self.num_hidden_layers						,					attention_heads=self.num_attention_heads						,					ffn_dim=self.ffn_dim						,					activation_function=self.activation_function						,					activation_dropout=self.activation_dropout						,					attention_dropout=self.attention_dropout						,					max_position_embeddings=self.max_position_embeddings						,					initializer_range=self.initializer_range						,					use_cache=_A						,					bos_token_id=self.bos_token_id						,					eos_token_id=self.eos_token_id						,					pad_token_id=self.pad_token_id						,					)

       return (config, input_ids, input_mask)






 def 						lowercase_    (      self	:Optional[Any]      )					->      str:







       '''simple docstring'''



       __A													=      self.prepare_config_and_inputs()
       __A						,   __A						,   __A													=      config_and_inputs
       __A													=      {'input_ids': input_ids, 'attention_mask': attention_mask}
       return config, inputs_dict






 def 						lowercase_    (      self	:str      )					->      int:







       '''simple docstring'''



       __A						,   __A						,   __A													=      self.prepare_config_and_inputs()

       __A													=      floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
       __A													=      ids_tensor([self.batch_size, self.seq_length]						,					vocab_size=2      )

       return (
           config,
           input_ids,
           attention_mask,
           encoder_hidden_states,
           encoder_attention_mask,
       )






 def 						lowercase_    (      self	:Union[str, Any]						,					_A	:Optional[int]						,					_A	:List[Any]						,					_A	:str						,					_A	:Optional[Any]      )					->      List[Any]:







       '''simple docstring'''



       __A													=      20
       __A													=      model_class_name(_A      )

       __A													=      model.init_cache(input_ids.shape[0]						,					_A      )
       __A													=      jnp.ones((input_ids.shape[0], max_decoder_length)						,					dtype='i4'      )

       __A													=      jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1      )[None, :]						,					(input_ids.shape[0], input_ids.shape[-1] - 1)      )
       __A													=      model(
           input_ids[:, :-1]						,					attention_mask=_A						,					past_key_values=_A						,					position_ids=_A						,					)

       __A													=      jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,					dtype='i4'      )
       __A													=      model(
           input_ids[:, -1:]						,					attention_mask=_A						,					past_key_values=outputs_cache.past_key_values						,					position_ids=_A						,					)

       __A													=      model(_A      )

       __A													=      np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])      )      )
       self.parent.assertTrue(diff < 1E-3						,					msg=F'Max diff is {diff}'      )






 def 						lowercase_    (      self	:int						,					_A	:Union[str, Any]						,					_A	:int						,					_A	:Optional[int]						,					_A	:Tuple      )					->      List[str]:







       '''simple docstring'''



       __A													=      20
       __A													=      model_class_name(_A      )

       __A													=      jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])      )]						,					axis=-1						,					)

       __A													=      model.init_cache(input_ids.shape[0]						,					_A      )
       __A													=      jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1      )[None, :]						,					(input_ids.shape[0], input_ids.shape[-1] - 1)      )

       __A													=      model(
           input_ids[:, :-1]						,					attention_mask=_A						,					past_key_values=_A						,					position_ids=_A						,					)
       __A													=      jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,					dtype='i4'      )
       __A													=      model(
           input_ids[:, -1:]						,					past_key_values=outputs_cache.past_key_values						,					attention_mask=_A						,					position_ids=_A						,					)

       __A													=      model(_A						,					attention_mask=_A      )
       __A													=      np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])      )      )
       self.parent.assertTrue(diff < 1E-3						,					msg=F'Max diff is {diff}'      )


@require_sentencepiece
@require_flax
class 			UpperCamelCase__	(     SCREAMING_SNAKE_CASE   ,  SCREAMING_SNAKE_CASE   ,  unittest.TestCase):
 UpperCAmelCase__     :     Any    					=       (FlaxXGLMModel, FlaxXGLMForCausalLM) if is_flax_available() else ()
 UpperCAmelCase__     :     Optional[Any]    					=       (FlaxXGLMForCausalLM,) if is_flax_available() else ()






 def 						lowercase_    (      self	:Dict      )					->      Any:







       '''simple docstring'''



       __A													=      FlaxXGLMModelTester(self      )






 def 						lowercase_    (      self	:Dict      )					->      List[str]:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
             __A						,   __A						,   __A													=      self.model_tester.prepare_config_and_inputs()
             self.model_tester.check_use_cache_forward(_A						,					_A						,					_A						,					_A      )






 def 						lowercase_    (      self	:str      )					->      int:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
             __A						,   __A						,   __A													=      self.model_tester.prepare_config_and_inputs()
             self.model_tester.check_use_cache_forward_with_attn_mask(
                 _A						,					_A						,					_A						,					_A      )






 @slow
 def 						lowercase_    (      self	:Tuple      )					->      List[str]:







       '''simple docstring'''



       __A													=      XGLMTokenizer.from_pretrained('XGLM'						,					padding_side='left'      )
       __A													=      tokenizer(['Hello this is a long string', 'Hey']						,					return_tensors='np'						,					padding=_A						,					truncation=_A      )

       __A													=      FlaxXGLMForCausalLM.from_pretrained('facebook/xglm-564M'      )
       __A													=      1
       __A													=      False

       __A													=      jax.jit(model.generate      )

       __A													=      jit_generate(inputs['input_ids']						,					attention_mask=inputs['attention_mask']      ).sequences

       __A													=      tokenizer.batch_decode(_A						,					skip_special_tokens=_A      )

       __A													=      [
           'Hello this is a long string of questions, but I\'m not sure if I\'m',
           'Hey, I\'m a newbie to the forum and I\'',
       ]

       self.assertListEqual(_A						,					_A      )






 @is_pt_flax_cross_test
 def 						lowercase_    (      self	:int      )					->      Optional[Any]:







       '''simple docstring'''



       __A						,   __A													=      self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
             with self.subTest(model_class.__name__      ):
                   # prepare inputs
                   __A													=      self._prepare_for_class(_A						,					_A      )
                   __A													=      {k: torch.tensor(v.tolist()      ) for k, v in prepared_inputs_dict.items()}

                   # load corresponding PyTorch class
                   __A													=      model_class.__name__[4:]  # Skip the "Flax" at the beginning
                   __A													=      getattr(_A						,					_A      )

                   __A						,   __A													=      pt_inputs['input_ids'].shape
                   __A													=      np.random.randint(0						,					seq_length - 1						,					size=(batch_size,)      )
                   for batch_idx, start_index in enumerate(_A      ):
                         __A													=      0
                         __A													=      1
                         __A													=      0
                         __A													=      1
                   __A													=      pt_model_class(_A      ).eval()
                   # Flax models don't use the `use_cache` option and cache is not returned as a default.
                   # So we disable `use_cache` here for PyTorch model.
                   __A													=      False
                   __A													=      model_class(_A						,					dtype=jnp.floataa      )
                   __A													=      convert_pytorch_state_dict_to_flax(pt_model.state_dict()						,					_A      )
                   __A													=      fx_state

                   with torch.no_grad():
                         __A													=      pt_model(**_A      ).to_tuple()

                   __A													=      fx_model(**_A      ).to_tuple()
                   self.assertEqual(len(_A      )						,					len(_A      )						,					'Output lengths differ between Flax and PyTorch'      )
                   for fx_output, pt_output in zip(_A						,					_A      ):
                         self.assert_almost_equals(fx_output[:, -1]						,					pt_output[:, -1].numpy()						,					4E-2      )

                   with tempfile.TemporaryDirectory() as tmpdirname:
                         pt_model.save_pretrained(_A      )
                         __A													=      model_class.from_pretrained(_A						,					from_pt=_A      )

                   __A													=      fx_model_loaded(**_A      ).to_tuple()
                   self.assertEqual(
                       len(_A      )						,					len(_A      )						,					'Output lengths differ between Flax and PyTorch'      )
                   for fx_output_loaded, pt_output in zip(_A						,					_A      ):
                         self.assert_almost_equals(fx_output_loaded[:, -1]						,					pt_output[:, -1].numpy()						,					4E-2      )






 @is_pt_flax_cross_test
 def 						lowercase_    (      self	:List[str]      )					->      Optional[Any]:







       '''simple docstring'''



       __A						,   __A													=      self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
             with self.subTest(model_class.__name__      ):
                   # prepare inputs
                   __A													=      self._prepare_for_class(_A						,					_A      )
                   __A													=      {k: torch.tensor(v.tolist()      ) for k, v in prepared_inputs_dict.items()}

                   # load corresponding PyTorch class
                   __A													=      model_class.__name__[4:]  # Skip the "Flax" at the beginning
                   __A													=      getattr(_A						,					_A      )

                   __A													=      pt_model_class(_A      ).eval()
                   __A													=      False
                   __A													=      model_class(_A						,					dtype=jnp.floataa      )

                   __A													=      load_flax_weights_in_pytorch_model(_A						,					fx_model.params      )
                   __A						,   __A													=      pt_inputs['input_ids'].shape
                   __A													=      np.random.randint(0						,					seq_length - 1						,					size=(batch_size,)      )
                   for batch_idx, start_index in enumerate(_A      ):
                         __A													=      0
                         __A													=      1
                         __A													=      0
                         __A													=      1

                   # make sure weights are tied in PyTorch
                   pt_model.tie_weights()

                   with torch.no_grad():
                         __A													=      pt_model(**_A      ).to_tuple()

                   __A													=      fx_model(**_A      ).to_tuple()
                   self.assertEqual(len(_A      )						,					len(_A      )						,					'Output lengths differ between Flax and PyTorch'      )
                   for fx_output, pt_output in zip(_A						,					_A      ):
                         self.assert_almost_equals(fx_output[:, -1]						,					pt_output[:, -1].numpy()						,					4E-2      )

                   with tempfile.TemporaryDirectory() as tmpdirname:
                         fx_model.save_pretrained(_A      )
                         __A													=      pt_model_class.from_pretrained(_A						,					from_flax=_A      )

                   with torch.no_grad():
                         __A													=      pt_model_loaded(**_A      ).to_tuple()

                   self.assertEqual(
                       len(_A      )						,					len(_A      )						,					'Output lengths differ between Flax and PyTorch'      )
                   for fx_output, pt_output in zip(_A						,					_A      ):
                         self.assert_almost_equals(fx_output[:, -1]						,					pt_output[:, -1].numpy()						,					4E-2      )






 @slow
 def 						lowercase_    (      self	:Union[str, Any]      )					->      Union[str, Any]:







       '''simple docstring'''



       for model_class_name in self.all_model_classes:
             __A													=      model_class_name.from_pretrained('facebook/xglm-564M'      )
             __A													=      model(np.ones((1, 1)      )      )
             self.assertIsNotNone(_A      )








"""simple docstring"""







import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__     =						logging.get_logger(__name__)

lowerCamelCase__     =						{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCamelCase__     =						{
    """vocab_file""": {
        """microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json""",
    },
    """merges_file""": {"""microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"""},
}

lowerCamelCase__     =						{
    """microsoft/biogpt""": 1_024,
}
def        __lowerCAmelCase							(_UpperCamelCase						):
	__lowerCAmelCase     :		Tuple		     = set()
	__lowerCAmelCase     :		str		     = word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)						)
		__lowerCAmelCase     :		Optional[Any]		     = char
	return pairs




class 					A__			(      _lowerCamelCase):
		A_		:			Union[str, Any]							  =      VOCAB_FILES_NAMES
		A_		:			List[Any]							  =      PRETRAINED_VOCAB_FILES_MAP
		A_		:			Optional[Any]							  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A_		:			int							  =      ['input_ids', 'attention_mask']

		def __init__( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE="<unk>"			,						_SCREAMING_SNAKE_CASE="<s>"			,						_SCREAMING_SNAKE_CASE="</s>"			,						_SCREAMING_SNAKE_CASE="</s>"			,						_SCREAMING_SNAKE_CASE="<pad>"			,						**_SCREAMING_SNAKE_CASE			,						):
			super().__init__(
			    bos_token=_SCREAMING_SNAKE_CASE			,						eos_token=_SCREAMING_SNAKE_CASE			,						sep_token=_SCREAMING_SNAKE_CASE			,						unk_token=_SCREAMING_SNAKE_CASE			,						pad_token=_SCREAMING_SNAKE_CASE			,						**_SCREAMING_SNAKE_CASE			,						)

			try:
				import sacremoses
			except ImportError:
				raise ImportError(
				    'You need to install sacremoses to use BioGptTokenizer. '
				    'See https://pypi.org/project/sacremoses/ for installation.'					)

			__lowerCAmelCase     :		int		     = 'en'
			__lowerCAmelCase     :		Optional[Any]		     = sacremoses
			# cache of sm.MosesTokenizer instance
			__lowerCAmelCase     :		Optional[int]		     = {}
			__lowerCAmelCase     :		Any		     = {}
			with open(_SCREAMING_SNAKE_CASE			,						encoding='utf-8'					) as vocab_handle:
				__lowerCAmelCase     :		int		     = json.load(_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Any		     = {v: k for k, v in self.encoder.items()}
			with open(_SCREAMING_SNAKE_CASE			,						encoding='utf-8'					) as merges_handle:
				__lowerCAmelCase     :		Optional[int]		     = merges_handle.read().split('\n'					)[:-1]
			__lowerCAmelCase     :		Dict		     = [tuple(merge.split()[:2]					) for merge in merges]
			__lowerCAmelCase     :		Optional[Any]		     = dict(zip(_SCREAMING_SNAKE_CASE			,						range(len(_SCREAMING_SNAKE_CASE					)					)					)					)
			__lowerCAmelCase     :		Optional[int]		     = {}

		@property
		def 				__lowerCamelCase				( self					):
			return len(self.encoder					)

		def 				__lowerCamelCase				( self					):
			return dict(self.encoder			,						**self.added_tokens_encoder					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					):
			if lang not in self.cache_moses_tokenizer:
				__lowerCAmelCase     :		str		     = self.sm.MosesTokenizer(lang=_SCREAMING_SNAKE_CASE					)
				__lowerCAmelCase     :		Optional[int]		     = moses_tokenizer
			return self.cache_moses_tokenizer[lang].tokenize(
			    _SCREAMING_SNAKE_CASE			,						aggressive_dash_splits=_SCREAMING_SNAKE_CASE			,						return_str=_SCREAMING_SNAKE_CASE			,						escape=_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					):
			if lang not in self.cache_moses_detokenizer:
				__lowerCAmelCase     :		Any		     = self.sm.MosesDetokenizer(lang=_SCREAMING_SNAKE_CASE					)
				__lowerCAmelCase     :		Any		     = moses_detokenizer
			return self.cache_moses_detokenizer[lang].detokenize(_SCREAMING_SNAKE_CASE					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE					):
			__lowerCAmelCase     :		List[Any]		     = tuple(token[:-1]					) + (token[-1] + '</w>',)
			if token in self.cache:
				return self.cache[token]
			__lowerCAmelCase     :		Union[str, Any]		     = get_pairs(_SCREAMING_SNAKE_CASE					)

			if not pairs:
				return token + "</w>"

			while True:
				__lowerCAmelCase     :		List[Any]		     = min(_SCREAMING_SNAKE_CASE			,						key=lambda _SCREAMING_SNAKE_CASE					: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE			,						float('inf'					)					)					)
				if bigram not in self.bpe_ranks:
					break
				__lowerCAmelCase     ,  __lowerCAmelCase     :		str		     = bigram
				__lowerCAmelCase     :		Any		     = []
				__lowerCAmelCase     :		Optional[Any]		     = 0
				while i < len(_SCREAMING_SNAKE_CASE					):
					try:
						__lowerCAmelCase     :		Optional[Any]		     = word.index(_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE					)
					except ValueError:
						new_word.extend(word[i:]					)
						break
					else:
						new_word.extend(word[i:j]					)
						__lowerCAmelCase     :		int		     = j

					if word[i] == first and i < len(_SCREAMING_SNAKE_CASE					) - 1 and word[i + 1] == second:
						new_word.append(first + second					)
						i += 2
					else:
						new_word.append(word[i]					)
						i += 1
				__lowerCAmelCase     :		Optional[Any]		     = tuple(_SCREAMING_SNAKE_CASE					)
				__lowerCAmelCase     :		Optional[Any]		     = new_word
				if len(_SCREAMING_SNAKE_CASE					) == 1:
					break
				else:
					__lowerCAmelCase     :		Optional[Any]		     = get_pairs(_SCREAMING_SNAKE_CASE					)
			__lowerCAmelCase     :		Optional[int]		     = ' '.join(_SCREAMING_SNAKE_CASE					)
			if word == "\n  </w>":
				__lowerCAmelCase     :		Dict		     = '\n</w>'
			__lowerCAmelCase     :		Optional[int]		     = word
			return word

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE=False					):
			if bypass_tokenizer:
				__lowerCAmelCase     :		Any		     = text.split()
			else:
				__lowerCAmelCase     :		Union[str, Any]		     = self.moses_tokenize(_SCREAMING_SNAKE_CASE			,						self.lang					)

			__lowerCAmelCase     :		List[str]		     = []
			for token in text:
				if token:
					split_tokens.extend(list(self.bpe(_SCREAMING_SNAKE_CASE					).split(' '					)					)					)

			return split_tokens

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE					):
			return self.encoder.get(_SCREAMING_SNAKE_CASE			,						self.encoder.get(self.unk_token					)					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE					):
			return self.decoder.get(_SCREAMING_SNAKE_CASE			,						self.unk_token					)

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE					):
			__lowerCAmelCase     :		List[str]		     = [t.replace(' '			,						''					).replace('</w>'			,						' '					) for t in tokens]
			__lowerCAmelCase     :		List[Any]		     = ''.join(_SCREAMING_SNAKE_CASE					).split()
			# detokenize
			__lowerCAmelCase     :		int		     = self.moses_detokenize(_SCREAMING_SNAKE_CASE			,						self.lang					)
			return text

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None					):
			if token_ids_a is None:
				return [self.sep_token_id] + token_ids_a
			__lowerCAmelCase     :		Dict		     = [self.sep_token_id]
			return sep + token_ids_a + sep + token_ids_a

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None			,						_SCREAMING_SNAKE_CASE = False					):
			if already_has_special_tokens:
				return super().get_special_tokens_mask(
				    token_ids_a=_SCREAMING_SNAKE_CASE			,						token_ids_a=_SCREAMING_SNAKE_CASE			,						already_has_special_tokens=_SCREAMING_SNAKE_CASE					)
			# no bos used in fairseq
			if token_ids_a is not None:
				return [1] + ([0] * len(_SCREAMING_SNAKE_CASE					)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE					))
			return [1] + ([0] * len(_SCREAMING_SNAKE_CASE					))

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None					):
			__lowerCAmelCase     :		int		     = [self.sep_token_id]

			# no bos used in fairseq
			if token_ids_a is None:
				return len(token_ids_a + sep					) * [0]
			return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]

		def 				__lowerCamelCase				( self			,						_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE = None					):
			if not os.path.isdir(_SCREAMING_SNAKE_CASE					):
				logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
				return
			__lowerCAmelCase     :		List[Any]		     = os.path.join(
			    _SCREAMING_SNAKE_CASE			,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
			__lowerCAmelCase     :		List[str]		     = os.path.join(
			    _SCREAMING_SNAKE_CASE			,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

			with open(_SCREAMING_SNAKE_CASE			,						'w'			,						encoding='utf-8'					) as f:
				f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=_SCREAMING_SNAKE_CASE			,						ensure_ascii=_SCREAMING_SNAKE_CASE					) + '\n'					)

			__lowerCAmelCase     :		str		     = 0
			with open(_SCREAMING_SNAKE_CASE			,						'w'			,						encoding='utf-8'					) as writer:
				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda _SCREAMING_SNAKE_CASE					: kv[1]					):
					if index != token_index:
						logger.warning(
						    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
						    ' Please check that the tokenizer is not corrupted!'					)
						__lowerCAmelCase     :		List[Any]		     = token_index
					writer.write(' '.join(_SCREAMING_SNAKE_CASE					) + '\n'					)
					index += 1

			return vocab_file, merge_file

		def __getstate__( self					):
			__lowerCAmelCase     :		Any		     = self.__dict__.copy()
			__lowerCAmelCase     :		List[str]		     = None
			return state







		def __setstate__( self			,						_SCREAMING_SNAKE_CASE					):
			__lowerCAmelCase     :		List[str]		     = d

			try:
				import sacremoses
			except ImportError:
				raise ImportError(
				    'You need to install sacremoses to use XLMTokenizer. '
				    'See https://pypi.org/project/sacremoses/ for installation.'					)

			__lowerCAmelCase     :		int		     = sacremoses