


'''simple docstring'''



import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


__lowercase	:						str       =	{
    'cola': 2,
    'mnli': 3,
    'mrpc': 2,
    'sst-2': 2,
    'sts-b': 1,
    'qqp': 2,
    'qnli': 2,
    'rte': 2,
    'wnli': 2,
}


logging.set_verbosity_info()






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Optional[Any]						,      _SCREAMING_SNAKE_CASE      :   Optional[Any]						,      _SCREAMING_SNAKE_CASE      :   str						,      _SCREAMING_SNAKE_CASE      :   Tuple=None   ):
					# Initialise PyTorch model
					__a			:       Union[str, Any]					    =  XLNetConfig.from_json_file(_SCREAMING_SNAKE_CASE   )

					__a			:       str					    =  finetuning_task.lower() if finetuning_task is not None else ''
					if finetuning_task in GLUE_TASKS_NUM_LABELS:
										print(F"""Building PyTorch XLNetForSequenceClassification model from configuration: {config}"""   )
										__a			:       Tuple					    =  finetuning_task
										__a			:       Tuple					    =  GLUE_TASKS_NUM_LABELS[finetuning_task]
										__a			:       Union[str, Any]					    =  XLNetForSequenceClassification(_SCREAMING_SNAKE_CASE   )
					elif "squad" in finetuning_task:
										__a			:       List[str]					    =  finetuning_task
										__a			:       Tuple					    =  XLNetForQuestionAnswering(_SCREAMING_SNAKE_CASE   )
					else:
										__a			:       str					    =  XLNetLMHeadModel(_SCREAMING_SNAKE_CASE   )

					# Load weights from tf checkpoint
					load_tf_weights_in_xlnet(_SCREAMING_SNAKE_CASE						,      _SCREAMING_SNAKE_CASE						,      _SCREAMING_SNAKE_CASE   )

					# Save pytorch-model
					__a			:       Tuple					    =  os.path.join(_SCREAMING_SNAKE_CASE						,      _SCREAMING_SNAKE_CASE   )
					__a			:       Any					    =  os.path.join(_SCREAMING_SNAKE_CASE						,      _SCREAMING_SNAKE_CASE   )
					print(F"""Save PyTorch model to {os.path.abspath(_SCREAMING_SNAKE_CASE   )}"""   )
					torch.save(model.state_dict()						,      _SCREAMING_SNAKE_CASE   )
					print(F"""Save configuration file to {os.path.abspath(_SCREAMING_SNAKE_CASE   )}"""   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'						,      encoding='utf-8'   ) as f:
										f.write(config.to_json_string()   )


if __name__ == "__main__":
	__lowercase	:						Optional[int]       =	argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
	)
	parser.add_argument(
	    '--xlnet_config_file',
	    default=None,
	    type=str,
	    required=True,
	    help=(
	        'The config json file corresponding to the pre-trained XLNet model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the folder to store the PyTorch model or dataset/vocab.',
	)
	parser.add_argument(
	    '--finetuning_task',
	    default=None,
	    type=str,
	    help='Name of a task on which the XLNet TensorFlow model was fine-tuned',
	)
	__lowercase	:						List[str]       =	parser.parse_args()
	print(args)

	convert_xlnet_checkpoint_to_pytorch(
	    args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
	)







'''simple docstring'''



import os
import textwrap

import pyarrow as pa
import pytest

from datasets import ClassLabel, Features, Image
from datasets.packaged_modules.csv.csv import Csv

from ..utils import require_pil






@pytest.fixture
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Dict   ):
					__a			:       Optional[Any]					    =  tmp_path / 'file.csv'
					__a			:       Union[str, Any]					    =  textwrap.dedent(
					    '\\n        header1,header2\n        1,2\n        10,20\n        '   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'   ) as f:
										f.write(_SCREAMING_SNAKE_CASE   )
					return str(_SCREAMING_SNAKE_CASE   )






@pytest.fixture
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Union[str, Any]   ):
					__a			:       str					    =  tmp_path / 'malformed_file.csv'
					__a			:       int					    =  textwrap.dedent(
					    '\\n        header1,header2\n        1,2\n        10,20,\n        '   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'   ) as f:
										f.write(_SCREAMING_SNAKE_CASE   )
					return str(_SCREAMING_SNAKE_CASE   )






@pytest.fixture
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Optional[Any]						,      _SCREAMING_SNAKE_CASE      :   List[str]   ):
					__a			:       Optional[Any]					    =  tmp_path / 'csv_with_image.csv'
					__a			:       Dict					    =  textwrap.dedent(
					    F"""\
        image
        {image_file}
        """   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'   ) as f:
										f.write(_SCREAMING_SNAKE_CASE   )
					return str(_SCREAMING_SNAKE_CASE   )






@pytest.fixture
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Optional[Any]   ):
					__a			:       Union[str, Any]					    =  tmp_path / 'csv_with_label.csv'
					__a			:       Any					    =  textwrap.dedent(
					    '\\n        label\n        good\n        bad\n        good\n        '   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'   ) as f:
										f.write(_SCREAMING_SNAKE_CASE   )
					return str(_SCREAMING_SNAKE_CASE   )






@pytest.fixture
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   List[Any]   ):
					__a			:       Dict					    =  tmp_path / 'csv_with_int_list.csv'
					__a			:       Tuple					    =  textwrap.dedent(
					    '\\n        int_list\n        1 2 3\n        4 5 6\n        7 8 9\n        '   )
					with open(_SCREAMING_SNAKE_CASE						,      'w'   ) as f:
										f.write(_SCREAMING_SNAKE_CASE   )
					return str(_SCREAMING_SNAKE_CASE   )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   List[str]						,      _SCREAMING_SNAKE_CASE      :   int						,      _SCREAMING_SNAKE_CASE      :   List[str]   ):
					__a			:       int					    =  Csv()
					__a			:       str					    =  csv._generate_tables([[csv_file, malformed_csv_file]]   )
					with pytest.raises(_SCREAMING_SNAKE_CASE						,      match='Error tokenizing data'   ):
										for _ in generator:
															pass
					assert any(
					    record.levelname == 'ERROR'
					    and 'Failed to read file' in record.message
					    and os.path.basename(_SCREAMING_SNAKE_CASE   ) in record.message
					    for record in caplog.records   )






@require_pil
def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   List[str]   ):
					with open(_SCREAMING_SNAKE_CASE						,      encoding='utf-8'   ) as f:
										__a			:       Tuple					    =  f.read().splitlines()[1]
					__a			:       Tuple					    =  Csv(encoding='utf-8'						,      features=Features({'image': Image()}   )   )
					__a			:       Any					    =  csv._generate_tables([[csv_file_with_image]]   )
					__a			:       int					    =  pa.concat_tables([table for _, table in generator]   )
					assert pa_table.schema.field('image'   ).type == Image()()
					__a			:       Any					    =  pa_table.to_pydict()['image']
					assert generated_content == [{"path": image_file, "bytes": None}]






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Union[str, Any]   ):
					with open(_SCREAMING_SNAKE_CASE						,      encoding='utf-8'   ) as f:
										__a			:       Tuple					    =  f.read().splitlines()[1:]
					__a			:       Optional[int]					    =  Csv(encoding='utf-8'						,      features=Features({'label': ClassLabel(names=['good', 'bad']   )}   )   )
					__a			:       List[str]					    =  csv._generate_tables([[csv_file_with_label]]   )
					__a			:       Dict					    =  pa.concat_tables([table for _, table in generator]   )
					assert pa_table.schema.field('label'   ).type == ClassLabel(names=['good', 'bad']   )()
					__a			:       int					    =  pa_table.to_pydict()['label']
					assert generated_content == [ClassLabel(names=['good', 'bad']   ).straint(_SCREAMING_SNAKE_CASE   ) for label in labels]






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE      :   Optional[int]   ):
					__a			:       str					    =  Csv(encoding='utf-8'						,      sep=','						,      converters={'int_list': lambda _SCREAMING_SNAKE_CASE   : [int(_SCREAMING_SNAKE_CASE   ) for i in x.split()]}   )
					__a			:       Any					    =  csv._generate_tables([[csv_file_with_int_list]]   )
					__a			:       Any					    =  pa.concat_tables([table for _, table in generator]   )
					assert pa.types.is_list(pa_table.schema.field('int_list'   ).type   )
					__a			:       Tuple					    =  pa_table.to_pydict()['int_list']
					assert generated_content == [[1, 2, 3], [4, 5, 6], [7, 8, 9]]



