
import inspect
import unittest

import requests

from transformers import MgpstrConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import MgpstrForSceneTextRecognition


if is_vision_available():
							from PIL import Image

							from transformers import MgpstrProcessor
class       A__  :





							def __init__(					self    :  Optional[int]	,					_UpperCAmelCase    :  int	,					_UpperCAmelCase    :  List[str]=False	,					_UpperCAmelCase    :  List[Any]=13	,					_UpperCAmelCase    :  Optional[int]=(32, 1_28)	,					_UpperCAmelCase    :  Union[str, Any]=4	,					_UpperCAmelCase    :  Dict=3	,					_UpperCAmelCase    :  Tuple=27	,					_UpperCAmelCase    :  Any=38	,					_UpperCAmelCase    :  List[str]=99	,					_UpperCAmelCase    :  str=99	,					_UpperCAmelCase    :  List[Any]=32	,					_UpperCAmelCase    :  Any=5	,					_UpperCAmelCase    :  Tuple=4	,					_UpperCAmelCase    :  Any=4.0	,					_UpperCAmelCase    :  List[Any]=2_57	,					_UpperCAmelCase    :  int=None	,					)		->						Dict:



											"""simple docstring"""


											__lowercase								=   parent
											__lowercase								=   is_training
											__lowercase								=   batch_size
											__lowercase								=   image_size
											__lowercase								=   patch_size
											__lowercase								=   num_channels
											__lowercase								=   max_token_length
											__lowercase								=   num_character_labels
											__lowercase								=   num_bpe_labels
											__lowercase								=   num_wordpiece_labels
											__lowercase								=   hidden_size
											__lowercase								=   num_hidden_layers
											__lowercase								=   num_attention_heads
											__lowercase								=   mlp_ratio
											__lowercase								=   patch_embeds_hidden_size
											__lowercase								=   output_hidden_states





							def      a__					(					self    :  Any							)		->						Optional[int]:



											"""simple docstring"""


											__lowercase								=   floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]							)
											__lowercase								=   self.get_config()
											return config, pixel_values





							def      a__					(					self    :  Any							)		->						Optional[int]:



											"""simple docstring"""


											return MgpstrConfig(
											    image_size=self.image_size	,					patch_size=self.patch_size	,					num_channels=self.num_channels	,					max_token_length=self.max_token_length	,					num_character_labels=self.num_character_labels	,					num_bpe_labels=self.num_bpe_labels	,					num_wordpiece_labels=self.num_wordpiece_labels	,					hidden_size=self.hidden_size	,					num_hidden_layers=self.num_hidden_layers	,					num_attention_heads=self.num_attention_heads	,					mlp_ratio=self.mlp_ratio	,					output_hidden_states=self.output_hidden_states	,					)





							def      a__					(					self    :  Union[str, Any]	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  Union[str, Any]							)		->						List[str]:



											"""simple docstring"""


											__lowercase								=   MgpstrForSceneTextRecognition(_UpperCAmelCase							)
											model.to(_UpperCAmelCase							)
											model.eval()
											with torch.no_grad():
															__lowercase								=   model(_UpperCAmelCase							)
											self.parent.assertEqual(
											    generated_ids[0][0].shape	,					(self.batch_size, self.max_token_length, self.num_character_labels)							)





							def      a__					(					self    :  Optional[Any]							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase								=   self.prepare_config_and_inputs()
											__lowercase						,   __lowercase								=   config_and_inputs
											__lowercase								=   {'pixel_values': pixel_values}
											return config, inputs_dict






@require_torch
class       A__  (		lowerCAmelCase__       ,		lowerCAmelCase__       ,		unittest.TestCase							):
							lowerCAmelCase__						:		List[str]            =				(MgpstrForSceneTextRecognition,) if is_torch_available() else ()
							lowerCAmelCase__						:		str            =				{"feature-extraction": MgpstrForSceneTextRecognition} if is_torch_available() else {}
							lowerCAmelCase__						:		Dict            =				False

							lowerCAmelCase__						:		List[str]            =				False
							lowerCAmelCase__						:		Optional[int]            =				False
							lowerCAmelCase__						:		List[Any]            =				False
							lowerCAmelCase__						:		Any            =				False





							def      a__					(					self    :  Dict							)		->						int:



											"""simple docstring"""


											__lowercase								=   MgpstrModelTester(self							)
											__lowercase								=   ConfigTester(self	,					config_class=_UpperCAmelCase	,					has_text_modality=_UpperCAmelCase							)





							def      a__					(					self    :  List[str]							)		->						Dict:



											"""simple docstring"""


											self.config_tester.run_common_tests()





							def      a__					(					self    :  str							)		->						str:



											"""simple docstring"""


											__lowercase								=   self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*_UpperCAmelCase							)





							@unittest.skip(reason='MgpstrModel does not use inputs_embeds'							)
							def      a__					(					self    :  Tuple							)		->						List[str]:



											"""simple docstring"""


											pass





							def      a__					(					self    :  Optional[int]							)		->						Tuple:



											"""simple docstring"""


											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															__lowercase								=   model_class(_UpperCAmelCase							)
															self.assertIsInstance(model.get_input_embeddings()	,					(nn.Module)							)
															__lowercase								=   model.get_output_embeddings()
															self.assertTrue(x is None or isinstance(_UpperCAmelCase	,					nn.Linear							)							)





							def      a__					(					self    :  List[str]							)		->						Union[str, Any]:



											"""simple docstring"""


											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															__lowercase								=   model_class(_UpperCAmelCase							)
															__lowercase								=   inspect.signature(model.forward							)
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															__lowercase								=   [*signature.parameters.keys()]

															__lowercase								=   ['pixel_values']
															self.assertListEqual(arg_names[:1]	,					_UpperCAmelCase							)





							@unittest.skip(reason='MgpstrModel does not support feedforward chunking'							)
							def      a__					(					self    :  int							)		->						Union[str, Any]:



											"""simple docstring"""


											pass





							def      a__					(					self    :  List[str]							)		->						Optional[Any]:



											"""simple docstring"""


											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															if not model_class.supports_gradient_checkpointing:
																			continue

															__lowercase								=   True
															__lowercase								=   model_class(_UpperCAmelCase							)
															self.assertTrue(model.is_gradient_checkpointing							)





							def      a__					(					self    :  List[str]							)		->						Dict:



											"""simple docstring"""


											def check_hidden_states_output(_UpperCAmelCase    :  Any	,					_UpperCAmelCase    :  Optional[int]	,					_UpperCAmelCase    :  Tuple							):
															__lowercase								=   model_class(_UpperCAmelCase							)
															model.to(_UpperCAmelCase							)
															model.eval()

															with torch.no_grad():
																			__lowercase								=   model(**self._prepare_for_class(_UpperCAmelCase	,					_UpperCAmelCase							)							)

															__lowercase								=   outputs.hidden_states

															__lowercase								=   getattr(
															    self.model_tester	,					'expected_num_hidden_layers'	,					self.model_tester.num_hidden_layers + 1							)
															self.assertEqual(len(_UpperCAmelCase							)	,					_UpperCAmelCase							)

															self.assertListEqual(
															    list(hidden_states[0].shape[-2:]							)	,					[self.model_tester.patch_embeds_hidden_size, self.model_tester.hidden_size]	,					)

											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
															__lowercase								=   True
															check_hidden_states_output(_UpperCAmelCase	,					_UpperCAmelCase	,					_UpperCAmelCase							)

															# check that output_hidden_states also work using config
															del inputs_dict["output_hidden_states"]
															__lowercase								=   True

															check_hidden_states_output(_UpperCAmelCase	,					_UpperCAmelCase	,					_UpperCAmelCase							)





							def      a__					(					self    :  int							)		->						Dict:



											"""simple docstring"""


											__lowercase						,   __lowercase								=   self.model_tester.prepare_config_and_inputs_for_common()

											__lowercase								=   _config_zero_init(_UpperCAmelCase							)
											for model_class in self.all_model_classes:
															__lowercase								=   model_class(config=_UpperCAmelCase							)
															for name, param in model.named_parameters():
																			if isinstance(_UpperCAmelCase	,					(nn.Linear, nn.Convad, nn.LayerNorm)							):
																							if param.requires_grad:
																											self.assertIn(
																											    ((param.data.mean() * 1e9).round() / 1e9).item()	,					[0.0, 1.0]	,					msg=f"""Parameter {name} of model {model_class} seems not properly initialized"""	,					)





							@unittest.skip(reason='Retain_grad is tested in individual model tests'							)
							def      a__					(					self    :  Optional[int]							)		->						int:



											"""simple docstring"""


											pass






def 				__SCREAMING_SNAKE_CASE  (						)	->		Dict:
				__lowercase								=   'https://i.postimg.cc/ZKwLg2Gw/367-14.png'
				__lowercase								=   Image.open(requests.get(SCREAMING_SNAKE_CASE       ,				stream=SCREAMING_SNAKE_CASE     ).raw     ).convert('RGB'     )
				return im
@require_vision
@require_torch
class       A__  (		unittest.TestCase							):





							@slow
							def      a__					(					self    :  List[Any]							)		->						int:



											"""simple docstring"""


											__lowercase								=   'alibaba-damo/mgp-str-base'
											__lowercase								=   MgpstrForSceneTextRecognition.from_pretrained(_UpperCAmelCase							).to(_UpperCAmelCase							)
											__lowercase								=   MgpstrProcessor.from_pretrained(_UpperCAmelCase							)

											__lowercase								=   prepare_img()
											__lowercase								=   processor(images=_UpperCAmelCase	,					return_tensors='pt'							).pixel_values.to(_UpperCAmelCase							)

											# forward pass
											with torch.no_grad():
															__lowercase								=   model(_UpperCAmelCase							)

											# verify the logits
											self.assertEqual(outputs.logits[0].shape	,					torch.Size((1, 27, 38)							)							)

											__lowercase								=   processor.batch_decode(outputs.logits							)
											__lowercase								=   'ticket'

											self.assertEqual(out_strs['generated_text'][0]	,					_UpperCAmelCase							)

											__lowercase								=   torch.tensor(
											    [[[-39.5_397, -44.4_024, -36.1_844], [-61.4_709, -63.8_639, -58.3_454], [-74.0_225, -68.5_494, -71.2_164]]]	,					device=_UpperCAmelCase	,					)

											self.assertTrue(torch.allclose(outputs.logits[0][:, 1:4, 1:4]	,					_UpperCAmelCase	,					atol=1e-4							)							)







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


SCREAMING_SNAKE_CASE__				=							datasets.utils.logging.get_logger(__name__)
@dataclass
class       A__  (		datasets.BuilderConfig							):
							lowerCAmelCase__						:		Optional[datasets.Features]            =				None
							lowerCAmelCase__						:		str            =				"utf-8"
							lowerCAmelCase__						:		Optional[str]            =				None
							lowerCAmelCase__						:		int            =				10 << 20  # 10MB
							lowerCAmelCase__						:		bool            =				False
							lowerCAmelCase__						:		str            =				"line"






class       A__  (		datasets.ArrowBasedBuilder							):
							lowerCAmelCase__						:		Optional[int]            =				TextConfig





							def      a__					(					self    :  List[str]							)		->						Any:



											"""simple docstring"""


											return datasets.DatasetInfo(features=self.config.features							)





							def      a__					(					self    :  Tuple	,					_UpperCAmelCase    :  str							)		->						List[str]:



											"""simple docstring"""


											if not self.config.data_files:
															raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
											__lowercase								=   dl_manager.download_and_extract(self.config.data_files							)
											if isinstance(_UpperCAmelCase	,					(str, list, tuple)							):
															__lowercase								=   data_files
															if isinstance(_UpperCAmelCase	,					_UpperCAmelCase							):
																			__lowercase								=   [files]
															__lowercase								=   [dl_manager.iter_files(_UpperCAmelCase							) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,					gen_kwargs={'files': files}							)]
											__lowercase								=   []
											for split_name, files in data_files.items():
															if isinstance(_UpperCAmelCase	,					_UpperCAmelCase							):
																			__lowercase								=   [files]
															__lowercase								=   [dl_manager.iter_files(_UpperCAmelCase							) for file in files]
															splits.append(datasets.SplitGenerator(name=_UpperCAmelCase	,					gen_kwargs={'files': files}							)							)
											return splits





							def      a__					(					self    :  int	,					_UpperCAmelCase    :  pa.Table							)		->						pa.Table:



											"""simple docstring"""


											if self.config.features is not None:
															__lowercase								=   self.config.features.arrow_schema
															if all(not require_storage_cast(_UpperCAmelCase							) for feature in self.config.features.values()							):
																			# cheaper cast
																			__lowercase								=   pa_table.cast(_UpperCAmelCase							)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			__lowercase								=   table_cast(_UpperCAmelCase	,					_UpperCAmelCase							)
															return pa_table
											else:
															return pa_table.cast(pa.schema({'text': pa.string()}							)							)





							def      a__					(					self    :  Union[str, Any]	,					_UpperCAmelCase    :  Optional[Any]							)		->						List[Any]:



											"""simple docstring"""


											__lowercase								=   list(self.config.features							) if self.config.features is not None else ['text']
											for file_idx, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase							)							):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(_UpperCAmelCase	,					encoding=self.config.encoding	,					errors=self.config.errors							) as f:
																			if self.config.sample_by == "line":
																							__lowercase								=   0
																							while True:
																											__lowercase								=   f.read(self.config.chunksize							)
																											if not batch:
																															break
																											batch += f.readline()  # finish current line
																											# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																											__lowercase								=   StringIO(_UpperCAmelCase							).readlines()
																											if not self.config.keep_linebreaks:
																															__lowercase								=   [line.rstrip('\n'							) for line in batch]
																											__lowercase								=   pa.Table.from_arrays([pa.array(_UpperCAmelCase							)]	,					names=_UpperCAmelCase							)
																											# Uncomment for debugging (will print the Arrow table size and elements)
																											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																											batch_idx += 1
																			elif self.config.sample_by == "paragraph":
																							__lowercase								=   0
																							__lowercase								=   ''
																							while True:
																											__lowercase								=   f.read(self.config.chunksize							)
																											if not new_batch:
																															break
																											batch += new_batch
																											batch += f.readline()  # finish current line
																											__lowercase								=   batch.split('\n\n'							)
																											__lowercase								=   pa.Table.from_arrays(
																											    [pa.array([example for example in batch[:-1] if example]							)]	,					names=_UpperCAmelCase							)
																											# Uncomment for debugging (will print the Arrow table size and elements)
																											# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																											# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																											batch_idx += 1
																											__lowercase								=   batch[-1]
																							if batch:
																											__lowercase								=   pa.Table.from_arrays([pa.array([batch]							)]	,					names=_UpperCAmelCase							)
																											yield (file_idx, batch_idx), self._cast_table(_UpperCAmelCase							)
																			elif self.config.sample_by == "document":
																							__lowercase								=   f.read()
																							__lowercase								=   pa.Table.from_arrays([pa.array([text]							)]	,					names=_UpperCAmelCase							)
																							yield file_idx, self._cast_table(_UpperCAmelCase							)





