



'''simple docstring'''






import os
import textwrap

import pyarrow as pa
import pytest

from datasets import ClassLabel, Features, Image
from datasets.packaged_modules.csv.csv import Csv

from ..utils import require_pil




@pytest.fixture
def     snake_case_					(     lowerCAmelCase_		)->		List[Any]:





       '''simple docstring'''


       _UpperCAmelCase  :						Optional[Any]			     = tmp_path / """file.csv"""
       _UpperCAmelCase  :						Dict			     = textwrap.dedent(
           """\
        header1,header2
        1,2
        10,20
        """		)
       with open(SCREAMING_SNAKE_CASE__       ,   """w"""		) as f:
              f.write(SCREAMING_SNAKE_CASE__		)
       return str(SCREAMING_SNAKE_CASE__		)




@pytest.fixture
def     snake_case_					(     lowerCAmelCase_		)->		List[str]:





       '''simple docstring'''


       _UpperCAmelCase  :						Union[str, Any]			     = tmp_path / """malformed_file.csv"""
       _UpperCAmelCase  :						int			     = textwrap.dedent(
           """\
        header1,header2
        1,2
        10,20,
        """		)
       with open(SCREAMING_SNAKE_CASE__       ,   """w"""		) as f:
              f.write(SCREAMING_SNAKE_CASE__		)
       return str(SCREAMING_SNAKE_CASE__		)




@pytest.fixture
def     snake_case_					(     lowerCAmelCase_       ,   lowerCAmelCase_		)->		List[str]:





       '''simple docstring'''


       _UpperCAmelCase  :						Optional[Any]			     = tmp_path / """csv_with_image.csv"""
       _UpperCAmelCase  :						List[Any]			     = textwrap.dedent(
           F'''\
        image
        {image_file}
        '''		)
       with open(SCREAMING_SNAKE_CASE__       ,   """w"""		) as f:
              f.write(SCREAMING_SNAKE_CASE__		)
       return str(SCREAMING_SNAKE_CASE__		)




@pytest.fixture
def     snake_case_					(     lowerCAmelCase_		)->		List[str]:





       '''simple docstring'''


       _UpperCAmelCase  :						Optional[Any]			     = tmp_path / """csv_with_label.csv"""
       _UpperCAmelCase  :						List[str]			     = textwrap.dedent(
           """\
        label
        good
        bad
        good
        """		)
       with open(SCREAMING_SNAKE_CASE__       ,   """w"""		) as f:
              f.write(SCREAMING_SNAKE_CASE__		)
       return str(SCREAMING_SNAKE_CASE__		)




@pytest.fixture
def     snake_case_					(     lowerCAmelCase_		)->		int:





       '''simple docstring'''


       _UpperCAmelCase  :						str			     = tmp_path / """csv_with_int_list.csv"""
       _UpperCAmelCase  :						int			     = textwrap.dedent(
           """\
        int_list
        1 2 3
        4 5 6
        7 8 9
        """		)
       with open(SCREAMING_SNAKE_CASE__       ,   """w"""		) as f:
              f.write(SCREAMING_SNAKE_CASE__		)
       return str(SCREAMING_SNAKE_CASE__		)




def     snake_case_					(     lowerCAmelCase_       ,   lowerCAmelCase_       ,   lowerCAmelCase_		)->		Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase  :						List[Any]			     = Csv()
       _UpperCAmelCase  :						List[Any]			     = csv._generate_tables([[csv_file, malformed_csv_file]]		)
       with pytest.raises(SCREAMING_SNAKE_CASE__       ,   match="""Error tokenizing data"""		):
              for _ in generator:
                     pass
       assert any(
           record.levelname == """ERROR"""
           and """Failed to read file""" in record.message
           and os.path.basename(SCREAMING_SNAKE_CASE__		) in record.message
           for record in caplog.records		)




@require_pil
def     snake_case_					(     lowerCAmelCase_		)->		Dict:





       '''simple docstring'''


       with open(SCREAMING_SNAKE_CASE__       ,   encoding="""utf-8"""		) as f:
              _UpperCAmelCase  :						int			     = f.read().splitlines()[1]
       _UpperCAmelCase  :						Optional[int]			     = Csv(encoding="""utf-8"""       ,   features=Features({"""image""": Image()}		)		)
       _UpperCAmelCase  :						Union[str, Any]			     = csv._generate_tables([[csv_file_with_image]]		)
       _UpperCAmelCase  :						Any			     = pa.concat_tables([table for _, table in generator]		)
       assert pa_table.schema.field("""image"""		).type == Image()()
       _UpperCAmelCase  :						Dict			     = pa_table.to_pydict()["""image"""]
       assert generated_content == [{"path": image_file, "bytes": None}]




def     snake_case_					(     lowerCAmelCase_		)->		Dict:





       '''simple docstring'''


       with open(SCREAMING_SNAKE_CASE__       ,   encoding="""utf-8"""		) as f:
              _UpperCAmelCase  :						List[str]			     = f.read().splitlines()[1:]
       _UpperCAmelCase  :						str			     = Csv(encoding="""utf-8"""       ,   features=Features({"""label""": ClassLabel(names=["""good""", """bad"""]		)}		)		)
       _UpperCAmelCase  :						Any			     = csv._generate_tables([[csv_file_with_label]]		)
       _UpperCAmelCase  :						Tuple			     = pa.concat_tables([table for _, table in generator]		)
       assert pa_table.schema.field("""label"""		).type == ClassLabel(names=["""good""", """bad"""]		)()
       _UpperCAmelCase  :						Any			     = pa_table.to_pydict()["""label"""]
       assert generated_content == [ClassLabel(names=["""good""", """bad"""]		).straint(SCREAMING_SNAKE_CASE__		) for label in labels]




def     snake_case_					(     lowerCAmelCase_		)->		Optional[Any]:





       '''simple docstring'''


       _UpperCAmelCase  :						List[str]			     = Csv(encoding="""utf-8"""       ,   sep=""","""       ,   converters={"""int_list""": lambda lowerCAmelCase_		: [int(SCREAMING_SNAKE_CASE__		) for i in x.split()]}		)
       _UpperCAmelCase  :						str			     = csv._generate_tables([[csv_file_with_int_list]]		)
       _UpperCAmelCase  :						Optional[Any]			     = pa.concat_tables([table for _, table in generator]		)
       assert pa.types.is_list(pa_table.schema.field("""int_list"""		).type		)
       _UpperCAmelCase  :						Optional[Any]			     = pa_table.to_pydict()["""int_list"""]
       assert generated_content == [[1, 2, 3], [4, 5, 6], [7, 8, 9]]






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowerCAmelCase_       	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_       	=	{
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': 5_12,
}

lowerCAmelCase_       	=	{
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}





class 	snake_case_   (		__A					):







			'''simple docstring'''

			SCREAMING_SNAKE_CASE     :   Optional[Any]									=				VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE     :   Any									=				PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE     :   List[Any]									=				PRETRAINED_INIT_CONFIGURATION
			SCREAMING_SNAKE_CASE     :   Optional[int]									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE     :   Any									=				LxmertTokenizer







			def __init__(     self :				Union[str, Any]							,   _UpperCamelCase :				int=None							,   _UpperCamelCase :				Optional[Any]=None							,   _UpperCamelCase :				Dict=True							,   _UpperCamelCase :				Any="[UNK]"							,   _UpperCamelCase :				Tuple="[SEP]"							,   _UpperCamelCase :				List[Any]="[PAD]"							,   _UpperCamelCase :				Union[str, Any]="[CLS]"							,   _UpperCamelCase :				str="[MASK]"							,   _UpperCamelCase :				List[str]=True							,   _UpperCamelCase :				List[str]=None							,   **_UpperCamelCase :				List[str]							,   )      ->Any:
									super().__init__(
									    _UpperCamelCase							,   tokenizer_file=_UpperCamelCase							,   do_lower_case=_UpperCamelCase							,   unk_token=_UpperCamelCase							,   sep_token=_UpperCamelCase							,   pad_token=_UpperCamelCase							,   cls_token=_UpperCamelCase							,   mask_token=_UpperCamelCase							,   tokenize_chinese_chars=_UpperCamelCase							,   strip_accents=_UpperCamelCase							,   **_UpperCamelCase							,   )

									snake_case_   =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
									if (
									    normalizer_state.get('''lowercase'''							,   _UpperCamelCase							) != do_lower_case
									    or normalizer_state.get('''strip_accents'''							,   _UpperCamelCase							) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''							,   _UpperCamelCase							) != tokenize_chinese_chars
									):
															snake_case_   =	getattr(_UpperCamelCase							,   normalizer_state.pop('''type'''							)							)
															snake_case_   =	do_lower_case
															snake_case_   =	strip_accents
															snake_case_   =	tokenize_chinese_chars
															snake_case_   =	normalizer_class(**_UpperCamelCase							)

									snake_case_   =	do_lower_case







			def 				snake_case__(     self :				Optional[int]							,   _UpperCamelCase :				List[Any]							,   _UpperCamelCase :				List[str]=None							)      ->List[Any]:
									snake_case_   =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output







			def 				snake_case__(     self :				int							,   _UpperCamelCase :				List[int]							,   _UpperCamelCase :				Optional[List[int]] = None							)      ->List[int]:
									snake_case_   =	[self.sep_token_id]
									snake_case_   =	[self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







			def 				snake_case__(     self :				Any							,   _UpperCamelCase :				str							,   _UpperCamelCase :				Optional[str] = None							)      ->Tuple[str]:
									snake_case_   =	self._tokenizer.model.save(_UpperCamelCase							,   name=_UpperCamelCase							)
									return tuple(_UpperCamelCase							)