


'''simple docstring'''

import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_UpperCAmelCase					:			Union[str, Any]				     =					datasets.utils.logging.get_logger(__name__)


@dataclass
class    a__					(			datasets.BuilderConfig   ):







				"""simple docstring"""



				__UpperCamelCase     :		Optional[datasets.Features]						=						None
				__UpperCamelCase     :		str						=						"utf-8"
				__UpperCamelCase     :		Optional[str]						=						None
				__UpperCamelCase     :		int						=						10 << 20  # 10MB
				__UpperCamelCase     :		bool						=						False
				__UpperCamelCase     :		str						=						"line"


class    a__					(			datasets.ArrowBasedBuilder   ):







				"""simple docstring"""



				__UpperCamelCase     :		List[Any]						=						TextConfig




				def        _snake_case	(self		):
										return datasets.DatasetInfo(features=self.config.features		)




				def        _snake_case	(self							,      __lowercase		):
										if not self.config.data_files:
																raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""		)
										__lowerCAmelCase											=					dl_manager.download_and_extract(self.config.data_files		)
										if isinstance(__lowercase							,      (str, list, tuple)		):
																__lowerCAmelCase											=					data_files
																if isinstance(__lowercase							,      __lowercase		):
																						__lowerCAmelCase											=					[files]
																__lowerCAmelCase											=					[dl_manager.iter_files(__lowercase		) for file in files]
																return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,      gen_kwargs={'''files''': files}		)]
										__lowerCAmelCase											=					[]
										for split_name, files in data_files.items():
																if isinstance(__lowercase							,      __lowercase		):
																						__lowerCAmelCase											=					[files]
																__lowerCAmelCase											=					[dl_manager.iter_files(__lowercase		) for file in files]
																splits.append(datasets.SplitGenerator(name=__lowercase							,      gen_kwargs={'''files''': files}		)		)
										return splits




				def        _snake_case	(self							,      __lowercase		):
										if self.config.features is not None:
																__lowerCAmelCase											=					self.config.features.arrow_schema
																if all(not require_storage_cast(__lowercase		) for feature in self.config.features.values()		):
																						# cheaper cast
																						__lowerCAmelCase											=					pa_table.cast(__lowercase		)
																else:
																						# more expensive cast; allows str <-> int/float or str to Audio for example
																						__lowerCAmelCase											=					table_cast(__lowercase							,      __lowercase		)
																return pa_table
										else:
																return pa_table.cast(pa.schema({'''text''': pa.string()}		)		)




				def        _snake_case	(self							,      __lowercase		):
										__lowerCAmelCase											=					list(self.config.features		) if self.config.features is not None else ['''text''']
										for file_idx, file in enumerate(itertools.chain.from_iterable(__lowercase		)		):
																# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
																with open(__lowercase							,      encoding=self.config.encoding							,      errors=self.config.errors		) as f:
																						if self.config.sample_by == "line":
																												__lowerCAmelCase											=					0
																												while True:
																																		__lowerCAmelCase											=					f.read(self.config.chunksize		)
																																		if not batch:
																																								break
																																		batch += f.readline()  # finish current line
																																		# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																		__lowerCAmelCase											=					StringIO(__lowercase		).readlines()
																																		if not self.config.keep_linebreaks:
																																								__lowerCAmelCase											=					[line.rstrip('''\n'''		) for line in batch]
																																		__lowerCAmelCase											=					pa.Table.from_arrays([pa.array(__lowercase		)]							,      names=__lowercase		)
																																		# Uncomment for debugging (will print the Arrow table size and elements)
																																		# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																		# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																		yield (file_idx, batch_idx), self._cast_table(__lowercase		)
																																		batch_idx += 1
																						elif self.config.sample_by == "paragraph":
																												__lowerCAmelCase											=					0
																												__lowerCAmelCase											=					''''''
																												while True:
																																		__lowerCAmelCase											=					f.read(self.config.chunksize		)
																																		if not new_batch:
																																								break
																																		batch += new_batch
																																		batch += f.readline()  # finish current line
																																		__lowerCAmelCase											=					batch.split('''\n\n'''		)
																																		__lowerCAmelCase											=					pa.Table.from_arrays(
																																		    [pa.array([example for example in batch[:-1] if example]		)]							,      names=__lowercase		)
																																		# Uncomment for debugging (will print the Arrow table size and elements)
																																		# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																		# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																		yield (file_idx, batch_idx), self._cast_table(__lowercase		)
																																		batch_idx += 1
																																		__lowerCAmelCase											=					batch[-1]
																												if batch:
																																		__lowerCAmelCase											=					pa.Table.from_arrays([pa.array([batch]		)]							,      names=__lowercase		)
																																		yield (file_idx, batch_idx), self._cast_table(__lowercase		)
																						elif self.config.sample_by == "document":
																												__lowerCAmelCase											=					f.read()
																												__lowerCAmelCase											=					pa.Table.from_arrays([pa.array([text]		)]							,      names=__lowercase		)
																												yield file_idx, self._cast_table(__lowercase		)





'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import is_detectrona_available, logging


_UpperCAmelCase					:			Optional[int]				     =					logging.get_logger(__name__)

_UpperCAmelCase					:			str				     =					{
    """layoutlmv2-base-uncased""": """https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/config.json""",
    """layoutlmv2-large-uncased""": """https://huggingface.co/microsoft/layoutlmv2-large-uncased/resolve/main/config.json""",
    # See all LayoutLMv2 models at https://huggingface.co/models?filter=layoutlmv2
}

# soft dependency
if is_detectrona_available():
	import detectrona


class    a__					(			__A   ):







				"""simple docstring"""



				__UpperCamelCase     :		List[Any]						=						'layoutlmv2'




				def __init__(self							,      __lowercase=3_05_22							,      __lowercase=7_68							,      __lowercase=12							,      __lowercase=12							,      __lowercase=30_72							,      __lowercase="gelu"							,      __lowercase=0.1							,      __lowercase=0.1							,      __lowercase=5_12							,      __lowercase=2							,      __lowercase=0.0_2							,      __lowercase=1e-12							,      __lowercase=0							,      __lowercase=10_24							,      __lowercase=1_28							,      __lowercase=32							,      __lowercase=True							,      __lowercase=2_56							,      __lowercase=64							,      __lowercase=True							,      __lowercase=[7, 7, 2_56]							,      __lowercase=1_28							,      __lowercase=1_28							,      __lowercase=True							,      __lowercase=True							,      __lowercase=False							,      __lowercase=None							,      **__lowercase							,      ):
										super().__init__(
										    vocab_size=__lowercase							,      hidden_size=__lowercase							,      num_hidden_layers=__lowercase							,      num_attention_heads=__lowercase							,      intermediate_size=__lowercase							,      hidden_act=__lowercase							,      hidden_dropout_prob=__lowercase							,      attention_probs_dropout_prob=__lowercase							,      max_position_embeddings=__lowercase							,      type_vocab_size=__lowercase							,      initializer_range=__lowercase							,      layer_norm_eps=__lowercase							,      pad_token_id=__lowercase							,      **__lowercase							,      )
										__lowerCAmelCase											=					max_ad_position_embeddings
										__lowerCAmelCase											=					max_rel_pos
										__lowerCAmelCase											=					rel_pos_bins
										__lowerCAmelCase											=					fast_qkv
										__lowerCAmelCase											=					max_rel_ad_pos
										__lowerCAmelCase											=					rel_ad_pos_bins
										__lowerCAmelCase											=					convert_sync_batchnorm
										__lowerCAmelCase											=					image_feature_pool_shape
										__lowerCAmelCase											=					coordinate_size
										__lowerCAmelCase											=					shape_size
										__lowerCAmelCase											=					has_relative_attention_bias
										__lowerCAmelCase											=					has_spatial_attention_bias
										__lowerCAmelCase											=					has_visual_segment_embedding
										__lowerCAmelCase											=					(
										    detectrona_config_args if detectrona_config_args is not None else self.get_default_detectrona_config()
										)




				@classmethod
				def        _snake_case	(self		):
										return {
										    "MODEL.MASK_ON": True,
										    "MODEL.PIXEL_STD": [5_7.3_7_5, 5_7.1_2_0, 5_8.3_9_5],
										    "MODEL.BACKBONE.NAME": "build_resnet_fpn_backbone",
										    "MODEL.FPN.IN_FEATURES": ["res2", "res3", "res4", "res5"],
										    "MODEL.ANCHOR_GENERATOR.SIZES": [[32], [64], [1_28], [2_56], [5_12]],
										    "MODEL.RPN.IN_FEATURES": ["p2", "p3", "p4", "p5", "p6"],
										    "MODEL.RPN.PRE_NMS_TOPK_TRAIN": 20_00,
										    "MODEL.RPN.PRE_NMS_TOPK_TEST": 10_00,
										    "MODEL.RPN.POST_NMS_TOPK_TRAIN": 10_00,
										    "MODEL.POST_NMS_TOPK_TEST": 10_00,
										    "MODEL.ROI_HEADS.NAME": "StandardROIHeads",
										    "MODEL.ROI_HEADS.NUM_CLASSES": 5,
										    "MODEL.ROI_HEADS.IN_FEATURES": ["p2", "p3", "p4", "p5"],
										    "MODEL.ROI_BOX_HEAD.NAME": "FastRCNNConvFCHead",
										    "MODEL.ROI_BOX_HEAD.NUM_FC": 2,
										    "MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION": 14,
										    "MODEL.ROI_MASK_HEAD.NAME": "MaskRCNNConvUpsampleHead",
										    "MODEL.ROI_MASK_HEAD.NUM_CONV": 4,
										    "MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION": 7,
										    "MODEL.RESNETS.DEPTH": 1_01,
										    "MODEL.RESNETS.SIZES": [[32], [64], [1_28], [2_56], [5_12]],
										    "MODEL.RESNETS.ASPECT_RATIOS": [[0.5, 1.0, 2.0]],
										    "MODEL.RESNETS.OUT_FEATURES": ["res2", "res3", "res4", "res5"],
										    "MODEL.RESNETS.NUM_GROUPS": 32,
										    "MODEL.RESNETS.WIDTH_PER_GROUP": 8,
										    "MODEL.RESNETS.STRIDE_IN_1X1": False,
										}




				def        _snake_case	(self		):
										__lowerCAmelCase											=					detectrona.config.get_cfg()
										for k, v in self.detectrona_config_args.items():
																__lowerCAmelCase											=					k.split('''.'''		)
																__lowerCAmelCase											=					detectrona_config
																for attribute in attributes[:-1]:
																						__lowerCAmelCase											=					getattr(__lowercase							,      __lowercase		)
																setattr(__lowercase							,      attributes[-1]							,      __lowercase		)

										return detectrona_config

