





"""simple docstring"""





import tempfile
import unittest

import transformers
from transformers import XGLMConfig, XGLMTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
						import jax
						import jax.numpy as jnp
						import numpy as np

						from transformers.modeling_flax_pytorch_utils import (
						    convert_pytorch_state_dict_to_flax,
						    load_flax_weights_in_pytorch_model,
						)
						from transformers.models.xglm.modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel


if is_torch_available():
						import torch





@require_flax
class 							_lowerCamelCase   :


						def __init__(self ,				__a ,				__a=14 ,				__a=7 ,				__a=True ,				__a=True ,				__a=True ,				__a=99 ,				__a=32 ,				__a=5 ,				__a=4 ,				__a=37 ,				__a="gelu" ,				__a=0.1 ,				__a=0.1 ,				__a=5_12 ,				__a=0.02 ,				__a=None ,				)			-> Optional[int]:
								UpperCamelCase =						parent
								UpperCamelCase =						batch_size
								UpperCamelCase =						seq_length
								UpperCamelCase =						is_training
								UpperCamelCase =						use_input_mask
								UpperCamelCase =						use_labels
								UpperCamelCase =						vocab_size
								UpperCamelCase =						d_model
								UpperCamelCase =						num_hidden_layers
								UpperCamelCase =						num_attention_heads
								UpperCamelCase =						ffn_dim
								UpperCamelCase =						activation_function
								UpperCamelCase =						activation_dropout
								UpperCamelCase =						attention_dropout
								UpperCamelCase =						max_position_embeddings
								UpperCamelCase =						initializer_range
								UpperCamelCase =						None
								UpperCamelCase =						0
								UpperCamelCase =						2
								UpperCamelCase =						1


						def      snake_case_ (self		)			-> Union[str, Any]:
								UpperCamelCase =						np.clip(ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size		) ,				3 ,				self.vocab_size		)

								UpperCamelCase =						None
								if self.use_input_mask:
										UpperCamelCase =						random_attention_mask([self.batch_size, self.seq_length]		)

								UpperCamelCase =						XGLMConfig(
								    vocab_size=self.vocab_size ,				d_model=self.hidden_size ,				num_layers=self.num_hidden_layers ,				attention_heads=self.num_attention_heads ,				ffn_dim=self.ffn_dim ,				activation_function=self.activation_function ,				activation_dropout=self.activation_dropout ,				attention_dropout=self.attention_dropout ,				max_position_embeddings=self.max_position_embeddings ,				initializer_range=self.initializer_range ,				use_cache=lowercase_ ,				bos_token_id=self.bos_token_id ,				eos_token_id=self.eos_token_id ,				pad_token_id=self.pad_token_id ,				)

								return (config, input_ids, input_mask)


						def      snake_case_ (self		)			-> Any:
								UpperCamelCase =						self.prepare_config_and_inputs()
								UpperCamelCase =						config_and_inputs
								UpperCamelCase =						{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
								return config, inputs_dict


						def      snake_case_ (self		)			-> List[str]:
								UpperCamelCase =						self.prepare_config_and_inputs()

								UpperCamelCase =						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
								UpperCamelCase =						ids_tensor([self.batch_size, self.seq_length] ,				vocab_size=2		)

								return (
								    config,
								    input_ids,
								    attention_mask,
								    encoder_hidden_states,
								    encoder_attention_mask,
								)


						def      snake_case_ (self ,				__a ,				__a ,				__a ,				__a		)			-> Optional[int]:
								UpperCamelCase =						20
								UpperCamelCase =						model_class_name(lowercase_		)

								UpperCamelCase =						model.init_cache(input_ids.shape[0] ,				lowercase_		)
								UpperCamelCase =						jnp.ones((input_ids.shape[0], max_decoder_length) ,				dtype="i4"		)

								UpperCamelCase =						jnp.broadcast_to(
								    jnp.arange(input_ids.shape[-1] - 1		)[None, :] ,				(input_ids.shape[0], input_ids.shape[-1] - 1)		)
								UpperCamelCase =						model(
								    input_ids[:, :-1] ,				attention_mask=lowercase_ ,				past_key_values=lowercase_ ,				position_ids=lowercase_ ,				)

								UpperCamelCase =						jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,				dtype="i4"		)
								UpperCamelCase =						model(
								    input_ids[:, -1:] ,				attention_mask=lowercase_ ,				past_key_values=outputs_cache.past_key_values ,				position_ids=lowercase_ ,				)

								UpperCamelCase =						model(lowercase_		)

								UpperCamelCase =						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
								self.parent.assertTrue(diff < 1e-3 ,				msg=F"Max diff is {diff}"		)




						def      snake_case_ (self ,				__a ,				__a ,				__a ,				__a		)			-> Union[str, Any]:
								UpperCamelCase =						20
								UpperCamelCase =						model_class_name(lowercase_		)

								UpperCamelCase =						jnp.concatenate(
								    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])		)] ,				axis=-1 ,				)

								UpperCamelCase =						model.init_cache(input_ids.shape[0] ,				lowercase_		)
								UpperCamelCase =						jnp.broadcast_to(
								    jnp.arange(input_ids.shape[-1] - 1		)[None, :] ,				(input_ids.shape[0], input_ids.shape[-1] - 1)		)

								UpperCamelCase =						model(
								    input_ids[:, :-1] ,				attention_mask=lowercase_ ,				past_key_values=lowercase_ ,				position_ids=lowercase_ ,				)
								UpperCamelCase =						jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]] ,				dtype="i4"		)
								UpperCamelCase =						model(
								    input_ids[:, -1:] ,				past_key_values=outputs_cache.past_key_values ,				attention_mask=lowercase_ ,				position_ids=lowercase_ ,				)

								UpperCamelCase =						model(lowercase_ ,				attention_mask=lowercase_		)
								UpperCamelCase =						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
								self.parent.assertTrue(diff < 1e-3 ,				msg=F"Max diff is {diff}"		)





@require_sentencepiece
@require_flax
class 							_lowerCamelCase   (							snake_case__	,      snake_case__	,      unittest.TestCase			):
						UpperCAmelCase_			=      (FlaxXGLMModel, FlaxXGLMForCausalLM) if is_flax_available() else ()
						UpperCAmelCase_			=      (FlaxXGLMForCausalLM,) if is_flax_available() else ()


						def      snake_case_ (self		)			-> Optional[int]:
								UpperCamelCase =						FlaxXGLMModelTester(self		)


						def      snake_case_ (self		)			-> Optional[Any]:
								for model_class_name in self.all_model_classes:
										UpperCamelCase =						self.model_tester.prepare_config_and_inputs()
										self.model_tester.check_use_cache_forward(lowercase_ ,				lowercase_ ,				lowercase_ ,				lowercase_		)


						def      snake_case_ (self		)			-> List[Any]:
								for model_class_name in self.all_model_classes:
										UpperCamelCase =						self.model_tester.prepare_config_and_inputs()
										self.model_tester.check_use_cache_forward_with_attn_mask(
										    lowercase_ ,				lowercase_ ,				lowercase_ ,				lowercase_		)


						@slow
						def      snake_case_ (self		)			-> Union[str, Any]:
								UpperCamelCase =						XGLMTokenizer.from_pretrained("XGLM" ,				padding_side="left"		)
								UpperCamelCase =						tokenizer(["Hello this is a long string", "Hey"] ,				return_tensors="np" ,				padding=lowercase_ ,				truncation=lowercase_		)

								UpperCamelCase =						FlaxXGLMForCausalLM.from_pretrained("facebook/xglm-564M"		)
								UpperCamelCase =						1
								UpperCamelCase =						False

								UpperCamelCase =						jax.jit(model.generate		)

								UpperCamelCase =						jit_generate(inputs["input_ids"] ,				attention_mask=inputs["attention_mask"]		).sequences

								UpperCamelCase =						tokenizer.batch_decode(lowercase_ ,				skip_special_tokens=lowercase_		)

								UpperCamelCase =						[
								    '''Hello this is a long string of questions, but I\'m not sure if I\'m''',
								    '''Hey, I\'m a newbie to the forum and I\'''',
								]

								self.assertListEqual(lowercase_ ,				lowercase_		)


						@is_pt_flax_cross_test
						def      snake_case_ (self		)			-> List[Any]:
								UpperCamelCase =						self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
										with self.subTest(model_class.__name__		):
												# prepare inputs
												UpperCamelCase =						self._prepare_for_class(lowercase_ ,				lowercase_		)
												UpperCamelCase =						{k: torch.tensor(v.tolist()		) for k, v in prepared_inputs_dict.items()}

												# load corresponding PyTorch class
												UpperCamelCase =						model_class.__name__[4:]  # Skip the "Flax" at the beginning
												UpperCamelCase =						getattr(lowercase_ ,				lowercase_		)

												UpperCamelCase =						pt_inputs['''input_ids'''].shape
												UpperCamelCase =						np.random.randint(0 ,				seq_length - 1 ,				size=(batch_size,)		)
												for batch_idx, start_index in enumerate(lowercase_		):
														UpperCamelCase =						0
														UpperCamelCase =						1
														UpperCamelCase =						0
														UpperCamelCase =						1
												UpperCamelCase =						pt_model_class(lowercase_		).eval()
												# Flax models don't use the `use_cache` option and cache is not returned as a default.
												# So we disable `use_cache` here for PyTorch model.
												UpperCamelCase =						False
												UpperCamelCase =						model_class(lowercase_ ,				dtype=jnp.floataa		)
												UpperCamelCase =						convert_pytorch_state_dict_to_flax(pt_model.state_dict() ,				lowercase_		)
												UpperCamelCase =						fx_state

												with torch.no_grad():
														UpperCamelCase =						pt_model(**lowercase_		).to_tuple()

												UpperCamelCase =						fx_model(**lowercase_		).to_tuple()
												self.assertEqual(len(lowercase_		) ,				len(lowercase_		) ,				"Output lengths differ between Flax and PyTorch"		)
												for fx_output, pt_output in zip(lowercase_ ,				lowercase_		):
														self.assert_almost_equals(fx_output[:, -1] ,				pt_output[:, -1].numpy() ,				4e-2		)

												with tempfile.TemporaryDirectory() as tmpdirname:
														pt_model.save_pretrained(lowercase_		)
														UpperCamelCase =						model_class.from_pretrained(lowercase_ ,				from_pt=lowercase_		)

												UpperCamelCase =						fx_model_loaded(**lowercase_		).to_tuple()
												self.assertEqual(
												    len(lowercase_		) ,				len(lowercase_		) ,				"Output lengths differ between Flax and PyTorch"		)
												for fx_output_loaded, pt_output in zip(lowercase_ ,				lowercase_		):
														self.assert_almost_equals(fx_output_loaded[:, -1] ,				pt_output[:, -1].numpy() ,				4e-2		)


						@is_pt_flax_cross_test
						def      snake_case_ (self		)			-> List[Any]:
								UpperCamelCase =						self.model_tester.prepare_config_and_inputs_for_common()
								for model_class in self.all_model_classes:
										with self.subTest(model_class.__name__		):
												# prepare inputs
												UpperCamelCase =						self._prepare_for_class(lowercase_ ,				lowercase_		)
												UpperCamelCase =						{k: torch.tensor(v.tolist()		) for k, v in prepared_inputs_dict.items()}

												# load corresponding PyTorch class
												UpperCamelCase =						model_class.__name__[4:]  # Skip the "Flax" at the beginning
												UpperCamelCase =						getattr(lowercase_ ,				lowercase_		)

												UpperCamelCase =						pt_model_class(lowercase_		).eval()
												UpperCamelCase =						False
												UpperCamelCase =						model_class(lowercase_ ,				dtype=jnp.floataa		)

												UpperCamelCase =						load_flax_weights_in_pytorch_model(lowercase_ ,				fx_model.params		)
												UpperCamelCase =						pt_inputs['''input_ids'''].shape
												UpperCamelCase =						np.random.randint(0 ,				seq_length - 1 ,				size=(batch_size,)		)
												for batch_idx, start_index in enumerate(lowercase_		):
														UpperCamelCase =						0
														UpperCamelCase =						1
														UpperCamelCase =						0
														UpperCamelCase =						1

												# make sure weights are tied in PyTorch
												pt_model.tie_weights()

												with torch.no_grad():
														UpperCamelCase =						pt_model(**lowercase_		).to_tuple()

												UpperCamelCase =						fx_model(**lowercase_		).to_tuple()
												self.assertEqual(len(lowercase_		) ,				len(lowercase_		) ,				"Output lengths differ between Flax and PyTorch"		)
												for fx_output, pt_output in zip(lowercase_ ,				lowercase_		):
														self.assert_almost_equals(fx_output[:, -1] ,				pt_output[:, -1].numpy() ,				4e-2		)

												with tempfile.TemporaryDirectory() as tmpdirname:
														fx_model.save_pretrained(lowercase_		)
														UpperCamelCase =						pt_model_class.from_pretrained(lowercase_ ,				from_flax=lowercase_		)

												with torch.no_grad():
														UpperCamelCase =						pt_model_loaded(**lowercase_		).to_tuple()

												self.assertEqual(
												    len(lowercase_		) ,				len(lowercase_		) ,				"Output lengths differ between Flax and PyTorch"		)
												for fx_output, pt_output in zip(lowercase_ ,				lowercase_		):
														self.assert_almost_equals(fx_output[:, -1] ,				pt_output[:, -1].numpy() ,				4e-2		)




						@slow
						def      snake_case_ (self		)			-> List[str]:
								for model_class_name in self.all_model_classes:
										UpperCamelCase =						model_class_name.from_pretrained("facebook/xglm-564M"		)
										UpperCamelCase =						model(np.ones((1, 1)		)		)
										self.assertIsNotNone(lowercase_		)









"""simple docstring"""





import argparse
import json
import re
import string
import sys

import numpy as np


_SCREAMING_SNAKE_CASE							:	str										=			0.5



def 							_lowerCAmelCase   (    UpperCAmelCase  :				str   ,				UpperCAmelCase  :				Optional[int]					):


					'''simple docstring'''






					UpperCamelCase__		:     Optional[Any]								=['''.''', ''',''', ''';''', ''':''']

					for token in remove_tokens:
										UpperCamelCase__		:     Optional[int]								=ground_truth.replace(UpperCAmelCase   ,				''''''					)
										UpperCamelCase__		:     List[Any]								=prediction.replace(UpperCAmelCase   ,				''''''					)

					UpperCamelCase__	,      UpperCamelCase__		:     List[Any]								=ground_truth.lower(), prediction.lower()
					UpperCamelCase__	,      UpperCamelCase__		:     Dict								=ground_truth.replace('''/'''   ,				''' '''					), prediction.replace('''/'''   ,				''' '''					)
					UpperCamelCase__	,      UpperCamelCase__		:     Any								=set(ground_truth.split(''' '''					)					), set(prediction.split(''' '''					)					)

					UpperCamelCase__		:     Optional[Any]								=ground_truth.intersection(UpperCAmelCase					)
					UpperCamelCase__		:     Optional[Any]								=ground_truth.union(UpperCAmelCase					)
					UpperCamelCase__		:     str								=len(UpperCAmelCase					) / len(UpperCAmelCase					)
					return jaccard



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Union[str, Any]					):


					'''simple docstring'''






					def remove_articles(UpperCAmelCase  :				int					):
										return re.sub(r'''\b(a|an|the)\b'''   ,				''' '''   ,				UpperCAmelCase					)

					def white_space_fix(UpperCAmelCase  :				Union[str, Any]					):
										return " ".join(text.split()					)

					def remove_punc(UpperCAmelCase  :				Optional[int]					):
										UpperCamelCase__		:     Dict								=set(string.punctuation					)
										return "".join(ch for ch in text if ch not in exclude					)

					def lower(UpperCAmelCase  :				Optional[Any]					):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase					)					)					)					)



def 							_lowerCAmelCase   (    UpperCAmelCase  :				int   ,				UpperCAmelCase  :				str   ,				UpperCAmelCase  :				Any					):


					'''simple docstring'''






					UpperCamelCase__	,      UpperCamelCase__	,      UpperCamelCase__		:     Optional[Any]								=0, 0, 0

					UpperCamelCase__		:     int								='''Parties''' in qa_id

					# first check if ground truth is empty
					if len(UpperCAmelCase					) == 0:
										if len(UpperCAmelCase					) > 0:
															fp += len(UpperCAmelCase					)  # false positive for each one
					else:
										for ground_truth in ground_truths:
															assert len(UpperCAmelCase					) > 0
															# check if there is a match
															UpperCamelCase__		:     Union[str, Any]								=False
															for pred in predictions:
																				if substr_ok:
																									UpperCamelCase__		:     List[str]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH or ground_truth in pred
																				else:
																									UpperCamelCase__		:     Union[str, Any]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH
																				if is_match:
																									UpperCamelCase__		:     Dict								=True

															if match_found:
																				tp += 1
															else:
																				fn += 1

        # now also get any fps by looping through preds
										for pred in predictions:
															# Check if there's a match. if so, don't count (don't want to double count based on the above)
															# but if there's no match, then this is a false positive.
															# (Note: we get the true positives in the above loop instead of this loop so that we don't double count
															# multiple predictions that are matched with the same answer.)
															UpperCamelCase__		:     Any								=False
															for ground_truth in ground_truths:
																				assert len(UpperCAmelCase					) > 0
																				if substr_ok:
																									UpperCamelCase__		:     Union[str, Any]								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH or ground_truth in pred
																				else:
																									UpperCamelCase__		:     Dict								=get_jaccard(UpperCAmelCase   ,				UpperCAmelCase					) >= IOU_THRESH
																				if is_match:
																									UpperCamelCase__		:     str								=True

															if not match_found:
																				fp += 1

					UpperCamelCase__		:     Union[str, Any]								=tp / (tp + fp) if tp + fp > 0 else np.nan
					UpperCamelCase__		:     Optional[int]								=tp / (tp + fn) if tp + fn > 0 else np.nan

					return precision, recall



def 							_lowerCAmelCase   (    UpperCAmelCase  :				List[Any]					):


					'''simple docstring'''






					UpperCamelCase__		:     Optional[Any]								=precisions[::-1]
					for i in range(1   ,				len(UpperCAmelCase					)					):
										UpperCamelCase__		:     int								=max(precision_best[i - 1]   ,				precision_best[i]					)
					UpperCamelCase__		:     str								=precision_best[::-1]
					return precisions



def 							_lowerCAmelCase   (    UpperCAmelCase  :				List[str]   ,				UpperCAmelCase  :				Any					):


					'''simple docstring'''






					UpperCamelCase__		:     int								=process_precisions(UpperCAmelCase					)
					UpperCamelCase__		:     str								=np.trapz(UpperCAmelCase   ,				UpperCAmelCase					)
					if np.isnan(UpperCAmelCase					):
										return 0
					return aupr



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Dict   ,				UpperCAmelCase  :				int   ,				UpperCAmelCase  :				int					):


					'''simple docstring'''






					UpperCamelCase__		:     List[Any]								=process_precisions(UpperCAmelCase					)
					UpperCamelCase__		:     Optional[int]								=0
					for prec, recall in zip(UpperCAmelCase   ,				UpperCAmelCase					):
										if recall >= recall_thresh:
															UpperCamelCase__		:     Any								=prec
															break
					return prec_at_recall



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Any   ,				UpperCAmelCase  :				Dict					):


					'''simple docstring'''






					return normalize_answer(UpperCAmelCase					) == normalize_answer(UpperCAmelCase					)



def 							_lowerCAmelCase   (    UpperCAmelCase  :				int   ,				UpperCAmelCase  :				Dict   ,				UpperCAmelCase  :				int					):


					'''simple docstring'''






					UpperCamelCase__		:     List[str]								=0
					for pred in predictions:
										for ground_truth in ground_truths:
															UpperCamelCase__		:     str								=metric_fn(UpperCAmelCase   ,				UpperCAmelCase					)
															if score == 1:  # break the loop when one prediction matches the ground truth
																				break
										if score == 1:
															break
					return score



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Optional[int]   ,				UpperCAmelCase  :				str					):


					'''simple docstring'''






					UpperCamelCase__		:     Any								=0
					UpperCamelCase__		:     List[Any]								=[]
					UpperCamelCase__		:     Optional[Any]								=[]
					for article in dataset:
										for paragraph in article["paragraphs"]:
															for qa in paragraph["qas"]:
																				total += 1
																				if qa["id"] not in predictions:
																									UpperCamelCase__		:     Union[str, Any]								='''Unanswered question ''' + qa['''id'''] + ''' will receive score 0.'''
																									print(UpperCAmelCase   ,				file=sys.stderr					)
																									continue
																				UpperCamelCase__		:     Union[str, Any]								=[x['''text'''] for x in qa['''answers''']]
																				UpperCamelCase__		:     Any								=predictions[qa['''id''']]
																				UpperCamelCase__	,      UpperCamelCase__		:     int								=compute_precision_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				qa['''id''']					)

																				precisions.append(UpperCAmelCase					)
																				recalls.append(UpperCAmelCase					)

																				if precision == 0 and recall == 0:
																									fa += 0
																				else:
																									fa += 2 * (precision * recall) / (precision + recall)

																				exact_match += metric_max_over_ground_truths(UpperCAmelCase   ,				UpperCAmelCase   ,				UpperCAmelCase					)

					UpperCamelCase__		:     Union[str, Any]								=[x for _, x in sorted(zip(UpperCAmelCase   ,				UpperCAmelCase					)					)]
					recalls.sort()

					UpperCamelCase__		:     Optional[int]								=100.0 * fa / total
					UpperCamelCase__		:     List[Any]								=100.0 * exact_match / total
					UpperCamelCase__		:     Dict								=get_aupr(UpperCAmelCase   ,				UpperCAmelCase					)

					UpperCamelCase__		:     Optional[Any]								=get_prec_at_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				recall_thresh=0.9					)
					UpperCamelCase__		:     Dict								=get_prec_at_recall(UpperCAmelCase   ,				UpperCAmelCase   ,				recall_thresh=0.8					)

					return {
					    "exact_match": exact_match,
					    "f1": fa,
					    "aupr": aupr,
					    "prec_at_80_recall": prec_at_aa_recall,
					    "prec_at_90_recall": prec_at_aa_recall,
					}


if __name__ == "__main__":
				_SCREAMING_SNAKE_CASE							:	str										=			argparse.ArgumentParser(description="""Evaluation for CUAD""")
				parser.add_argument("""dataset_file""", help="""Dataset file""")
				parser.add_argument("""prediction_file""", help="""Prediction File""")
				_SCREAMING_SNAKE_CASE							:	Dict										=			parser.parse_args()
				with open(args.dataset_file) as dataset_file:
								_SCREAMING_SNAKE_CASE							:	str										=			json.load(dataset_file)
								_SCREAMING_SNAKE_CASE							:	Tuple										=			dataset_json["""data"""]
				with open(args.prediction_file) as prediction_file:
								_SCREAMING_SNAKE_CASE							:	Dict										=			json.load(prediction_file)
				print(json.dumps(evaluate(dataset, predictions)))


