
"""simple docstring"""

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig






def 				A		(							snake_case :Dict							,   snake_case :Optional[int]							,   snake_case :int=0						) ->      Optional[int]:
 # Format the message.
 if name is None:
  __UpperCamelCase							    =			None
 else:
  __UpperCamelCase							    =			'.' * max(0							,   spaces - 2						) + '# {:' + str(5_0 - spaces						) + 's}'
  __UpperCamelCase							    =			fmt.format(lowerCAmelCase__						)

 # Print and recurse (if needed).
 if isinstance(lowerCAmelCase__							,   lowerCAmelCase__						):
  if msg is not None:
   print(lowerCAmelCase__						)
  for k in val.keys():
   recursive_print(lowerCAmelCase__							,   val[k]							,   spaces + 2						)
 elif isinstance(lowerCAmelCase__							,   torch.Tensor						):
  print(lowerCAmelCase__							,   ':'							,   val.size()						)
 else:
  print(lowerCAmelCase__							,   ':'							,   lowerCAmelCase__						)






def 				A		(							snake_case :Tuple							,   snake_case :str							,   snake_case :Any							,   snake_case :str							,   snake_case :Any						) ->      str:
 __UpperCamelCase							    =			param.size()
 if checkpoint_version == 1.0:
  # version 1.0 stores [num_heads * hidden_size * num_splits, :]
  __UpperCamelCase							    =			(num_heads, hidden_size, num_splits) + input_shape[1:]
  __UpperCamelCase							    =			param.view(*lowerCAmelCase__						)
  __UpperCamelCase							    =			param.transpose(0							,   2						)
  __UpperCamelCase							    =			param.transpose(1							,   2						).contiguous()
 elif checkpoint_version >= 2.0:
  # other versions store [num_heads * num_splits * hidden_size, :]
  __UpperCamelCase							    =			(num_heads, num_splits, hidden_size) + input_shape[1:]
  __UpperCamelCase							    =			param.view(*lowerCAmelCase__						)
  __UpperCamelCase							    =			param.transpose(0							,   1						).contiguous()
 __UpperCamelCase							    =			param.view(*lowerCAmelCase__						)
 return param






def 				A		(							snake_case :Dict							,   snake_case :List[str]							,   snake_case :str						) ->      Any:
 __UpperCamelCase							    =			{}

 # old versions did not store training args
 __UpperCamelCase							    =			input_state_dict.get('args'							,   lowerCAmelCase__						)
 if ds_args is not None:
  # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
  # from pprint import pprint
  # pprint(vars(ds_args))

  __UpperCamelCase							    =			ds_args.tokenizer_type
  __UpperCamelCase							    =			ds_args.padded_vocab_size
  __UpperCamelCase							    =			ds_args.max_position_embeddings
  __UpperCamelCase							    =			ds_args.hidden_size
  __UpperCamelCase							    =			ds_args.num_layers
  __UpperCamelCase							    =			ds_args.num_attention_heads
  __UpperCamelCase							    =			ds_args.ffn_hidden_size if 'ffn_hidden_size' in ds_args else 4 * ds_args.hidden_size
  # pprint(config)

 # The number of heads.
 __UpperCamelCase							    =			config.num_attention_heads
 # The hidden_size per head.
 __UpperCamelCase							    =			config.hidden_size // heads
 # Megatron-LM checkpoint version
 if "checkpoint_version" in input_state_dict.keys():
  __UpperCamelCase							    =			input_state_dict['checkpoint_version']
 else:
  __UpperCamelCase							    =			0.0

 # The model.
 __UpperCamelCase							    =			input_state_dict['model']
 # The language model.
 __UpperCamelCase							    =			model['language_model']
 # The embeddings.
 __UpperCamelCase							    =			lm['embedding']

 # The word embeddings.
 __UpperCamelCase							    =			embeddings['word_embeddings']['weight']
 # Truncate the embedding table to vocab_size rows.
 __UpperCamelCase							    =			word_embeddings[: config.vocab_size, :]
 # Store the word embeddings.
 __UpperCamelCase							    =			word_embeddings

 # The position embeddings.
 __UpperCamelCase							    =			embeddings['position_embeddings']['weight']
 assert pos_embeddings.size(0						) == config.max_position_embeddings and pos_embeddings.size(1						) == config.hidden_size
 # Store the position embeddings.
 __UpperCamelCase							    =			pos_embeddings

 # The token-type embeddings.
 __UpperCamelCase							    =			embeddings['tokentype_embeddings']['weight']
 # Store the position embeddings.
 __UpperCamelCase							    =			tokentype_embeddings

 # The transformer.
 __UpperCamelCase							    =			lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

 # The regex to extract layer names.
 __UpperCamelCase							    =			re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'						)

 # The simple map of names for "automated" rules.
 __UpperCamelCase							    =			{
     'attention.dense': '.attention.output.dense.',
     'self_attention.dense': '.attention.output.dense.',
     'mlp.dense_h_to_4h': '.intermediate.dense.',
     'mlp.dense_4h_to_h': '.output.dense.',
 }

 # Keep track of the attention/query/value tensor.
 __UpperCamelCase							    =			None

 # Extract the layers.
 for key, val in transformer.items():
  # Match the name.
  __UpperCamelCase							    =			layer_re.match(lowerCAmelCase__						)

  # Stop if that's not a layer
  if m is None:
   break

  # The index of the layer.
  __UpperCamelCase							    =			int(m.group(1						)						)
  # The name of the operation.
  __UpperCamelCase							    =			m.group(2						)
  # Is it a weight or a bias?
  __UpperCamelCase							    =			m.group(3						)

  # The name of the layer.
  __UpperCamelCase							    =			f'bert.encoder.layer.{layer_idx}'

  # For layernorm(s), simply store the layer norm.
  if op_name.endswith('layernorm'						):
   __UpperCamelCase							    =			'attention.ln' if op_name.startswith('input'						) else 'ln'
   __UpperCamelCase							    =			val

  # Transpose the QKV matrix.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "weight":
   # Make sure the QKV pointer is nil.
   assert attention_qkv_weight is None, ""

   __UpperCamelCase							    =			fix_query_key_value_ordering(lowerCAmelCase__							,   lowerCAmelCase__							,   3							,   lowerCAmelCase__							,   lowerCAmelCase__						)
   # Store the tensor as we need the bias as well to interleave QKV and biases.
   __UpperCamelCase							    =			out_val

  # Transpose the bias.
  elif (
      op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
  ) and weight_or_bias == "bias":
   # Make sure we read the weight tensor.
   assert attention_qkv_weight is not None, ""

   # Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
   __UpperCamelCase							    =			attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
   __UpperCamelCase							    =			attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
   __UpperCamelCase							    =			attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

   __UpperCamelCase							    =			fix_query_key_value_ordering(lowerCAmelCase__							,   lowerCAmelCase__							,   3							,   lowerCAmelCase__							,   lowerCAmelCase__						)
   # Split the bias.
   __UpperCamelCase							    =			out_val[0 * config.hidden_size : 1 * config.hidden_size]
   __UpperCamelCase							    =			out_val[1 * config.hidden_size : 2 * config.hidden_size]
   __UpperCamelCase							    =			out_val[2 * config.hidden_size : 3 * config.hidden_size]

   # Store.
   __UpperCamelCase							    =			q
   __UpperCamelCase							    =			q_bias
   __UpperCamelCase							    =			k
   __UpperCamelCase							    =			k_bias
   __UpperCamelCase							    =			v
   __UpperCamelCase							    =			v_bias

   # Clear the stored tensor.
   __UpperCamelCase							    =			None

  # Copy weights and biases as is.
  elif weight_or_bias in ["weight", "bias"]:
   __UpperCamelCase							    =			megatron_to_transformers[op_name]
   __UpperCamelCase							    =			val

    # The final layernorm.
 __UpperCamelCase							    =			transformer['final_layernorm.weight']
 __UpperCamelCase							    =			transformer['final_layernorm.bias']

 # The pooler.
 __UpperCamelCase							    =			lm['pooler']

 # Store the matrix and the bias.
 __UpperCamelCase							    =			pooler['dense.weight']
 __UpperCamelCase							    =			pooler['dense.bias']

 # The LM head from Megatron (for RACE).
 __UpperCamelCase							    =			model['lm_head']

 # The transform matrix.
 __UpperCamelCase							    =			lm_head['dense.weight']
 __UpperCamelCase							    =			lm_head['dense.bias']

 # The transform LN.
 __UpperCamelCase							    =			lm_head['layernorm.weight']
 __UpperCamelCase							    =			lm_head['layernorm.bias']

 # For the decoder, we replicate the weights.
 __UpperCamelCase							    =			word_embeddings
 __UpperCamelCase							    =			lm_head['bias']

 # The classifier from Megatron (for MLNI).
 __UpperCamelCase							    =			model['binary_head']

 # Store the classifier.
 __UpperCamelCase							    =			binary_head['weight']
 __UpperCamelCase							    =			binary_head['bias']

 # It should be done!
 return output_state_dict






def 				A		(							) ->      List[str]:
 __UpperCamelCase							    =			argparse.ArgumentParser()
 parser.add_argument('--print-checkpoint-structure'							,   action='store_true'						)
 parser.add_argument('path_to_checkpoint'							,   type=lowerCAmelCase__							,   help='Path to the ZIP file containing the checkpoint'						)
 parser.add_argument(
     '--config_file'							,   default=''							,   type=lowerCAmelCase__							,   help='An optional config json file describing the pre-trained model.'							,   )
 __UpperCamelCase							    =			parser.parse_args()

 # Extract the basename.
 __UpperCamelCase							    =			os.path.dirname(args.path_to_checkpoint						)

 # Load the model.
 # the .zip is very optional, let's keep it for backward compatibility
 print(f'Extracting PyTorch state dictionary from "{args.path_to_checkpoint}"'						)
 if args.path_to_checkpoint.endswith('.zip'						):
  with zipfile.ZipFile(args.path_to_checkpoint							,   'r'						) as checkpoint:
   with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'						) as pytorch_dict:
    __UpperCamelCase							    =			torch.load(lowerCAmelCase__							,   map_location='cpu'						)
 else:
  __UpperCamelCase							    =			torch.load(args.path_to_checkpoint							,   map_location='cpu'						)

 if args.config_file == "":
  # Default config of megatron-bert 345m
  __UpperCamelCase							    =			MegatronBertConfig()

  # different megatron-bert-*-345m models have different vocab sizes, so override the default
  # config (which is for megatron-bert-cased-345m) with the actual vocab dimension
  __UpperCamelCase							    =			input_state_dict['model']['lm_head']['bias'].numel()
 else:
  __UpperCamelCase							    =			MegatronBertConfig.from_json_file(args.config_file						)

 # Convert.
 print('Converting'						)
 __UpperCamelCase							    =			convert_megatron_checkpoint(lowerCAmelCase__							,   lowerCAmelCase__							,   lowerCAmelCase__						)

 # Print the structure of converted state dict.
 if args.print_checkpoint_structure:
  recursive_print(lowerCAmelCase__							,   lowerCAmelCase__						)

 # Store the config to file.
 print('Saving config'						)
 config.save_pretrained(lowerCAmelCase__						)

 # Store the state_dict to file.
 __UpperCamelCase							    =			os.path.join(lowerCAmelCase__							,   'pytorch_model.bin'						)
 print(f'Saving checkpoint to "{output_checkpoint_file}"'						)
 torch.save(lowerCAmelCase__							,   lowerCAmelCase__						)


####################################################################################################

if __name__ == "__main__":
      main()

####################################################################################################





import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__						:Tuple					    =       logging.get_logger(__name__)

lowercase__						:List[str]					    =       {
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

lowercase__						:List[str]					    =       {
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

lowercase__						:Dict					    =       {"stas/tiny-wmt19-en-de": 1024}
lowercase__						:str					    =       {
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}



def 			UpperCamelCase       (				lowerCAmelCase__						):
					'''simple docstring'''
					lowercase		      =		set()
					lowercase		      =		word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)						)
										lowercase		      =		char
					return pairs



def 			UpperCamelCase       (				lowerCAmelCase__						):
					'''simple docstring'''
					lowercase		      =		text.replace('''，'''					,     ''','''						)
					lowercase		      =		re.sub(R'''。\s*'''					,     '''. '''					,     lowerCAmelCase__						)
					lowercase		      =		text.replace('''、'''					,     ''','''						)
					lowercase		      =		text.replace('''”'''					,     '''"'''						)
					lowercase		      =		text.replace('''“'''					,     '''"'''						)
					lowercase		      =		text.replace('''∶'''					,     ''':'''						)
					lowercase		      =		text.replace('''：'''					,     ''':'''						)
					lowercase		      =		text.replace('''？'''					,     '''?'''						)
					lowercase		      =		text.replace('''《'''					,     '''"'''						)
					lowercase		      =		text.replace('''》'''					,     '''"'''						)
					lowercase		      =		text.replace('''）'''					,     ''')'''						)
					lowercase		      =		text.replace('''！'''					,     '''!'''						)
					lowercase		      =		text.replace('''（'''					,     '''('''						)
					lowercase		      =		text.replace('''；'''					,     ''';'''						)
					lowercase		      =		text.replace('''１'''					,     '''1'''						)
					lowercase		      =		text.replace('''」'''					,     '''"'''						)
					lowercase		      =		text.replace('''「'''					,     '''"'''						)
					lowercase		      =		text.replace('''０'''					,     '''0'''						)
					lowercase		      =		text.replace('''３'''					,     '''3'''						)
					lowercase		      =		text.replace('''２'''					,     '''2'''						)
					lowercase		      =		text.replace('''５'''					,     '''5'''						)
					lowercase		      =		text.replace('''６'''					,     '''6'''						)
					lowercase		      =		text.replace('''９'''					,     '''9'''						)
					lowercase		      =		text.replace('''７'''					,     '''7'''						)
					lowercase		      =		text.replace('''８'''					,     '''8'''						)
					lowercase		      =		text.replace('''４'''					,     '''4'''						)
					lowercase		      =		re.sub(R'''．\s*'''					,     '''. '''					,     lowerCAmelCase__						)
					lowercase		      =		text.replace('''～'''					,     '''~'''						)
					lowercase		      =		text.replace('''’'''					,     '''\''''						)
					lowercase		      =		text.replace('''…'''					,     '''...'''						)
					lowercase		      =		text.replace('''━'''					,     '''-'''						)
					lowercase		      =		text.replace('''〈'''					,     '''<'''						)
					lowercase		      =		text.replace('''〉'''					,     '''>'''						)
					lowercase		      =		text.replace('''【'''					,     '''['''						)
					lowercase		      =		text.replace('''】'''					,     ''']'''						)
					lowercase		      =		text.replace('''％'''					,     '''%'''						)
					return text



def 			UpperCamelCase       (				lowerCAmelCase__						):
					'''simple docstring'''
					lowercase		      =		[]
					for char in text:
										lowercase		      =		unicodedata.category(lowerCAmelCase__						)
										if cat.startswith('''C'''						):
															continue
										output.append(lowerCAmelCase__						)
					return "".join(lowerCAmelCase__						)







class 							lowercase						(   SCREAMING_SNAKE_CASE__						):
							lowercase_			:			Any			   =VOCAB_FILES_NAMES
							lowercase_			:			List[str]			   =PRETRAINED_VOCAB_FILES_MAP
							lowercase_			:			Any			   =PRETRAINED_INIT_CONFIGURATION
							lowercase_			:			List[str]			   =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_			:			Optional[Any]			   =['''input_ids''', '''attention_mask''']




							def __init__(							self  ,A__=None  ,A__=None  ,A__=None  ,A__=None  ,A__=False  ,A__="<unk>"  ,A__="<s>"  ,A__="</s>"  ,A__="<pad>"  ,**A__  ,):
												super().__init__(
												    langs=A__  ,src_vocab_file=A__  ,tgt_vocab_file=A__  ,merges_file=A__  ,do_lower_case=A__  ,unk_token=A__  ,bos_token=A__  ,sep_token=A__  ,pad_token=A__  ,**A__  ,)

												try:
																	import sacremoses
												except ImportError:
																	raise ImportError(
																	    '''You need to install sacremoses to use XLMTokenizer. '''
																	    '''See https://pypi.org/project/sacremoses/ for installation.''')

												lowercase		      =		sacremoses

												lowercase		      =		src_vocab_file
												lowercase		      =		tgt_vocab_file
												lowercase		      =		merges_file
												lowercase		      =		do_lower_case

												# cache of sm.MosesPunctNormalizer instance
												lowercase		      =		{}
												# cache of sm.MosesTokenizer instance
												lowercase		      =		{}
												lowercase		      =		{}

												if langs and len(A__) == 2:
																	lowercase      ,		lowercase		      =		langs
												else:
																	raise ValueError(
																	    f'arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '
																	    '''Usually that means that tokenizer can\'t find a mapping for the given model path '''
																	    '''in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.''')

												with open(A__  ,encoding='''utf-8''') as src_vocab_handle:
																	lowercase		      =		json.load(A__)
												with open(A__  ,encoding='''utf-8''') as tgt_vocab_handle:
																	lowercase		      =		json.load(A__)
																	lowercase		      =		{v: k for k, v in tgt_vocab.items()}
												with open(A__  ,encoding='''utf-8''') as merges_handle:
																	lowercase		      =		merges_handle.read().split('''\n''')[:-1]
												lowercase		      =		[tuple(merge.split()[:2]) for merge in merges]
												lowercase		      =		dict(zip(A__  ,range(len(A__))))
												lowercase		      =		{}




							def 					A__ (							self):
												return self.get_src_vocab()




							@property
							def 					A__ (							self):
												return self.src_vocab_size




							def 					A__ (							self  ,A__  ,A__):
												if lang not in self.cache_moses_punct_normalizer:
																	lowercase		      =		self.sm.MosesPunctNormalizer(lang=A__)
																	lowercase		      =		punct_normalizer
												return self.cache_moses_punct_normalizer[lang].normalize(A__)




							def 					A__ (							self  ,A__  ,A__):
												if lang not in self.cache_moses_tokenizer:
																	lowercase		      =		self.sm.MosesTokenizer(lang=A__)
																	lowercase		      =		moses_tokenizer
												return self.cache_moses_tokenizer[lang].tokenize(
												    A__  ,aggressive_dash_splits=A__  ,return_str=A__  ,escape=A__)




							def 					A__ (							self  ,A__  ,A__):
												if lang not in self.cache_moses_detokenizer:
																	lowercase		      =		self.sm.MosesDetokenizer(lang=A__)
																	lowercase		      =		moses_detokenizer
												return self.cache_moses_detokenizer[lang].detokenize(A__)




							def 					A__ (							self  ,A__  ,A__):
												lowercase		      =		replace_unicode_punct(A__)
												lowercase		      =		self.moses_punct_norm(A__  ,A__)
												lowercase		      =		remove_non_printing_char(A__)
												return text




							@property
							def 					A__ (							self):
												return len(self.encoder)




							@property
							def 					A__ (							self):
												return len(self.decoder)




							def 					A__ (							self):
												return dict(self.encoder  ,**self.added_tokens_encoder)




							def 					A__ (							self):
												return dict(self.decoder  ,**self.added_tokens_decoder)




							def 					A__ (							self  ,A__):
												lowercase		      =		tuple(token[:-1]) + (token[-1] + '''</w>''',)
												if token in self.cache:
																	return self.cache[token]
												lowercase		      =		get_pairs(A__)

												if not pairs:
																	return token + "</w>"

												while True:
																	lowercase		      =		min(A__  ,key=lambda A__: self.bpe_ranks.get(A__  ,float('''inf''')))
																	if bigram not in self.bpe_ranks:
																						break
																	lowercase      ,		lowercase		      =		bigram
																	lowercase		      =		[]
																	lowercase		      =		0
																	while i < len(A__):
																						try:
																											lowercase		      =		word.index(A__  ,A__)
																						except ValueError:
																											new_word.extend(word[i:])
																											break
																						else:
																											new_word.extend(word[i:j])
																											lowercase		      =		j

																						if word[i] == first and i < len(A__) - 1 and word[i + 1] == second:
																											new_word.append(first + second)
																											i += 2
																						else:
																											new_word.append(word[i])
																											i += 1
																	lowercase		      =		tuple(A__)
																	lowercase		      =		new_word
																	if len(A__) == 1:
																						break
																	else:
																						lowercase		      =		get_pairs(A__)
												lowercase		      =		''' '''.join(A__)
												if word == "\n  </w>":
																	lowercase		      =		'''\n</w>'''
												lowercase		      =		word
												return word




							def 					A__ (							self  ,A__  ,A__="en"  ,A__=False):
												lowercase		      =		self.src_lang

												if self.do_lower_case:
																	lowercase		      =		text.lower()

												if bypass_tokenizer:
																	lowercase		      =		text.split()
												else:
																	lowercase		      =		self.moses_pipeline(A__  ,lang=A__)
																	lowercase		      =		self.moses_tokenize(A__  ,lang=A__)

												lowercase		      =		[]
												for token in text:
																	if token:
																						split_tokens.extend(list(self.bpe(A__).split(''' ''')))

												return split_tokens




							def 					A__ (							self  ,A__):
												return self.encoder.get(A__  ,self.encoder.get(self.unk_token))




							def 					A__ (							self  ,A__):
												return self.decoder.get(A__  ,self.unk_token)




							def 					A__ (							self  ,A__):
												lowercase		      =		[t.replace(''' '''  ,'''''').replace('''</w>'''  ,''' ''') for t in tokens]
												lowercase		      =		''''''.join(A__).split()
												# detokenize
												lowercase		      =		self.moses_detokenize(A__  ,self.tgt_lang)
												return text




							def 					A__ (							self  ,A__  ,A__ = None):
												lowercase		      =		[self.sep_token_id]

												# no bos used in fairseq
												if token_ids_a is None:
																	return token_ids_a + sep
												return token_ids_a + sep + token_ids_a + sep




							def 					A__ (							self  ,A__  ,A__ = None  ,A__ = False):

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=A__  ,token_ids_a=A__  ,already_has_special_tokens=A__)
												# no bos used in fairseq
												if token_ids_a is not None:
																	return ([0] * len(A__)) + [1] + ([0] * len(A__)) + [1]
												return ([0] * len(A__)) + [1]




							def 					A__ (							self  ,A__  ,A__ = None):
												lowercase		      =		[self.sep_token_id]

												# no bos used in fairseq
												if token_ids_a is None:
																	return len(token_ids_a + sep) * [0]
												return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]




							def 					A__ (							self  ,A__  ,A__ = None):
												if not os.path.isdir(A__):
																	logger.error(f'Vocabulary path ({save_directory}) should be a directory')
																	return

												lowercase		      =		os.path.join(
												    A__  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''src_vocab_file'''])
												lowercase		      =		os.path.join(
												    A__  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''tgt_vocab_file'''])
												lowercase		      =		os.path.join(
												    A__  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

												with open(A__  ,'''w'''  ,encoding='''utf-8''') as f:
																	f.write(json.dumps(self.encoder  ,indent=2  ,sort_keys=A__  ,ensure_ascii=A__) + '''\n''')

												with open(A__  ,'''w'''  ,encoding='''utf-8''') as f:
																	lowercase		      =		{v: k for k, v in self.decoder.items()}
																	f.write(json.dumps(A__  ,indent=2  ,sort_keys=A__  ,ensure_ascii=A__) + '''\n''')

												lowercase		      =		0
												with open(A__  ,'''w'''  ,encoding='''utf-8''') as writer:
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,key=lambda A__: kv[1]):
																						if index != token_index:
																											logger.warning(
																											    f'Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'
																											    ''' Please check that the tokenizer is not corrupted!''')
																											lowercase		      =		token_index
																						writer.write(''' '''.join(A__) + '''\n''')
																						index += 1

												return src_vocab_file, tgt_vocab_file, merges_file




							def __getstate__(							self):
												lowercase		      =		self.__dict__.copy()
												lowercase		      =		None
												return state





							def __setstate__(							self  ,A__):
												lowercase		      =		d

												try:
																	import sacremoses
												except ImportError:
																	raise ImportError(
																	    '''You need to install sacremoses to use XLMTokenizer. '''
																	    '''See https://pypi.org/project/sacremoses/ for installation.''')

												lowercase		      =		sacremoses




