import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp

def 	__lowerCamelCase							(				__a    : List[Any]      ,							__a    : List[Any]      ,							__a    : int      ,							__a    : Optional[int]      ,							__a    : int = 4_096		)      ->       Optional[Any]:
	_lowercase   ,  _lowercase   ,  _lowercase   =key.shape[-3:]
	_lowercase   =value.shape[-1]
	_lowercase   =min(__a      ,							__a		)
	_lowercase   =query / jnp.sqrt(__a		)

	@functools.partial(jax.checkpoint      ,							prevent_cse=__a		)
	def summarize_chunk(__a    : Dict      ,							__a    : Optional[Any]      ,							__a    : Optional[int]		):
		_lowercase   =jnp.einsum("...qhd,...khd->...qhk"      ,							__a      ,							__a      ,							precision=__a		)

		_lowercase   =jnp.max(__a      ,							axis=-1      ,							keepdims=__a		)
		_lowercase   =jax.lax.stop_gradient(__a		)
		_lowercase   =jnp.exp(attn_weights - max_score		)

		_lowercase   =jnp.einsum("...vhf,...qhv->...qhf"      ,							__a      ,							__a      ,							precision=__a		)
		_lowercase   =jnp.einsum("...qhk->...qh"      ,							__a		)

		return (exp_values, exp_weights.sum(axis=-1		), max_score)

	def chunk_scanner(__a    : str		):
		# julienne key array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]      ,							slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]      ,							)

		# julienne value array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]      ,							slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]      ,							)

		return summarize_chunk(__a      ,							__a      ,							__a		)

	_lowercase   ,  _lowercase   ,  _lowercase   =jax.lax.map(f=__a      ,							xs=jnp.arange(0      ,							__a      ,							__a		)		)

	_lowercase   =jnp.max(__a      ,							axis=0      ,							keepdims=__a		)
	_lowercase   =jnp.exp(chunk_max - global_max		)

	chunk_values *= jnp.expand_dims(__a      ,							axis=-1		)
	chunk_weights *= max_diffs

	_lowercase   =chunk_values.sum(axis=0		)
	_lowercase   =jnp.expand_dims(__a      ,							-1		).sum(axis=0		)

	return all_values / all_weights

def 	__lowerCamelCase							(				__a    : Union[str, Any]      ,							__a    : Dict      ,							__a    : int      ,							__a    : Any=jax.lax.Precision.HIGHEST      ,							__a    : int = 1_024      ,							__a    : int = 4_096		)      ->       Any:
	_lowercase   ,  _lowercase   ,  _lowercase   =query.shape[-3:]

	def chunk_scanner(__a    : List[Any]      ,							__a    : Union[str, Any]		):
		# julienne query array
		_lowercase   =jax.lax.dynamic_slice(
		    operand=__a      ,							start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]      ,							slice_sizes=list(query.shape[:-3]		) + [min(__a      ,							__a		), num_heads, q_features]      ,							)

		return (
		    chunk_idx + query_chunk_size,  # unused ignore it
		    _query_chunk_attention(
		        query=__a      ,							key=__a      ,							value=__a      ,							precision=__a      ,							key_chunk_size=__a		),
		)

	_lowercase   ,  _lowercase   =jax.lax.scan(
	    f=__a      ,							init=0      ,							xs=__a      ,							length=math.ceil(num_q / query_chunk_size		)		)

	return jnp.concatenate(__a      ,							axis=-3		)  # fuse the chunked result back
class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					8
	__SCREAMING_SNAKE_CASE              =					64
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.dim_head * self.heads
		_lowercase   =self.dim_head**-0.5

		# Weights were exported with old names {to_q, to_k, to_v, to_out}
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_q"       )
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_k"       )
		_lowercase   =nn.Dense(lowerCAmelCase_       ,  use_bias=lowerCAmelCase_       ,  dtype=self.dtype       ,  name="to_v"       )

		_lowercase   =nn.Dense(self.query_dim       ,  dtype=self.dtype       ,  name="to_out_0"       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   ,  _lowercase   ,  _lowercase   =tensor.shape
		_lowercase   =self.heads
		_lowercase   =tensor.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  dim // head_size       )
		_lowercase   =jnp.transpose(lowerCAmelCase_       ,  (0, 2, 1, 3)       )
		_lowercase   =tensor.reshape(batch_size * head_size       ,  lowerCAmelCase_       ,  dim // head_size       )
		return tensor


	def __lowerCAmelCase      (   self       ,  lowerCAmelCase_       ):
		_lowercase   ,  _lowercase   ,  _lowercase   =tensor.shape
		_lowercase   =self.heads
		_lowercase   =tensor.reshape(batch_size // head_size       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
		_lowercase   =jnp.transpose(lowerCAmelCase_       ,  (0, 2, 1, 3)       )
		_lowercase   =tensor.reshape(batch_size // head_size       ,  lowerCAmelCase_       ,  dim * head_size       )
		return tensor


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=None       ,  lowerCAmelCase_=True       ):
		_lowercase   =hidden_states if context is None else context

		_lowercase   =self.query(lowerCAmelCase_       )
		_lowercase   =self.key(lowerCAmelCase_       )
		_lowercase   =self.value(lowerCAmelCase_       )

		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )
		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )
		_lowercase   =self.reshape_heads_to_batch_dim(lowerCAmelCase_       )

		if self.use_memory_efficient_attention:
			_lowercase   =query_states.transpose(1       ,  0       ,  2       )
			_lowercase   =key_states.transpose(1       ,  0       ,  2       )
			_lowercase   =value_states.transpose(1       ,  0       ,  2       )

			# this if statement create a chunk size for each layer of the unet
			# the chunk size is equal to the query_length dimension of the deepest layer of the unet

			_lowercase   =query_states.shape[-3]
			if flatten_latent_dim % 64 == 0:
				_lowercase   =int(flatten_latent_dim / 64       )
			elif flatten_latent_dim % 16 == 0:
				_lowercase   =int(flatten_latent_dim / 16       )
			elif flatten_latent_dim % 4 == 0:
				_lowercase   =int(flatten_latent_dim / 4       )
			else:
				_lowercase   =int(lowerCAmelCase_       )

			_lowercase   =jax_memory_efficient_attention(
			    lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  query_chunk_size=lowerCAmelCase_       ,  key_chunk_size=4096 * 4       )

			_lowercase   =hidden_states.transpose(1       ,  0       ,  2       )
		else:
			# compute attentions
			_lowercase   =jnp.einsum("b i d, b j d->b i j"       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
			_lowercase   =attention_scores * self.scale
			_lowercase   =nn.softmax(lowerCAmelCase_       ,  axis=2       )

			# attend to values
			_lowercase   =jnp.einsum("b i j, b j d -> b i d"       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )

		_lowercase   =self.reshape_batch_dim_to_heads(lowerCAmelCase_       )
		_lowercase   =self.proj_attn(lowerCAmelCase_       )
		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa
	__SCREAMING_SNAKE_CASE              =					False


	def __lowerCAmelCase      (   self       ):
		# self attention (or cross_attention if only_cross_attention is True)
		_lowercase   =FlaxAttention(
		    self.dim       ,  self.n_heads       ,  self.d_head       ,  self.dropout       ,  self.use_memory_efficient_attention       ,  dtype=self.dtype       )
		# cross attention
		_lowercase   =FlaxAttention(
		    self.dim       ,  self.n_heads       ,  self.d_head       ,  self.dropout       ,  self.use_memory_efficient_attention       ,  dtype=self.dtype       )
		_lowercase   =FlaxFeedForward(dim=self.dim       ,  dropout=self.dropout       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.LayerNorm(epsilon=1e-5       ,  dtype=self.dtype       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		# self attention
		_lowercase   =hidden_states
		if self.only_cross_attention:
			_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		else:
			_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		# cross attention
		_lowercase   =hidden_states
		_lowercase   =self.attna(self.norma(lowerCAmelCase_       )       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		# feed forward
		_lowercase   =hidden_states
		_lowercase   =self.ff(self.norma(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =hidden_states + residual

		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					1
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					False
	__SCREAMING_SNAKE_CASE              =					jnp.floataa
	__SCREAMING_SNAKE_CASE              =					False


	def __lowerCAmelCase      (   self       ):
		_lowercase   =nn.GroupNorm(num_groups=32       ,  epsilon=1e-5       )

		_lowercase   =self.n_heads * self.d_head
		if self.use_linear_projection:
			_lowercase   =nn.Dense(lowerCAmelCase_       ,  dtype=self.dtype       )
		else:
			_lowercase   =nn.Conv(
			    lowerCAmelCase_       ,  kernel_size=(1, 1)       ,  strides=(1, 1)       ,  padding="VALID"       ,  dtype=self.dtype       ,  )

		_lowercase   =[
		    FlaxBasicTransformerBlock(
		        lowerCAmelCase_       ,  self.n_heads       ,  self.d_head       ,  dropout=self.dropout       ,  only_cross_attention=self.only_cross_attention       ,  dtype=self.dtype       ,  use_memory_efficient_attention=self.use_memory_efficient_attention       ,  )
		    for _ in range(self.depth       )
		]

		if self.use_linear_projection:
			_lowercase   =nn.Dense(lowerCAmelCase_       ,  dtype=self.dtype       )
		else:
			_lowercase   =nn.Conv(
			    lowerCAmelCase_       ,  kernel_size=(1, 1)       ,  strides=(1, 1)       ,  padding="VALID"       ,  dtype=self.dtype       ,  )

		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   ,  _lowercase   ,  _lowercase   ,  _lowercase   =hidden_states.shape
		_lowercase   =hidden_states
		_lowercase   =self.norm(lowerCAmelCase_       )
		if self.use_linear_projection:
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  height * width       ,  lowerCAmelCase_       )
			_lowercase   =self.proj_in(lowerCAmelCase_       )
		else:
			_lowercase   =self.proj_in(lowerCAmelCase_       )
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  height * width       ,  lowerCAmelCase_       )

		for transformer_block in self.transformer_blocks:
			_lowercase   =transformer_block(lowerCAmelCase_       ,  lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

		if self.use_linear_projection:
			_lowercase   =self.proj_out(lowerCAmelCase_       )
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
		else:
			_lowercase   =hidden_states.reshape(lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       ,  lowerCAmelCase_       )
			_lowercase   =self.proj_out(lowerCAmelCase_       )

		_lowercase   =hidden_states + residual
		return self.dropout_layer(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		# The second linear layer needs to be called
		# net_2 for now to match the index of the Sequential layer
		_lowercase   =FlaxGEGLU(self.dim       ,  self.dropout       ,  self.dtype       )
		_lowercase   =nn.Dense(self.dim       ,  dtype=self.dtype       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   =self.net_a(lowerCAmelCase_       ,  deterministic=lowerCAmelCase_       )
		_lowercase   =self.net_a(lowerCAmelCase_       )
		return hidden_states

class 	_a  (							nn.Module					):



	"""simple docstring"""



	__SCREAMING_SNAKE_CASE              =					42
	__SCREAMING_SNAKE_CASE              =					0.0
	__SCREAMING_SNAKE_CASE              =					jnp.floataa


	def __lowerCAmelCase      (   self       ):
		_lowercase   =self.dim * 4
		_lowercase   =nn.Dense(inner_dim * 2       ,  dtype=self.dtype       )
		_lowercase   =nn.Dropout(rate=self.dropout       )


	def __call__(   self       ,  lowerCAmelCase_       ,  lowerCAmelCase_=True       ):
		_lowercase   =self.proj(lowerCAmelCase_       )
		_lowercase   ,  _lowercase   =jnp.split(lowerCAmelCase_       ,  2       ,  axis=2       )
		return self.dropout_layer(hidden_linear * nn.gelu(lowerCAmelCase_       )       ,  deterministic=lowerCAmelCase_       )









'''simple docstring'''







import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


lowercase__  :	str            =      logging.get_logger(__name__)

@dataclass
class    lowerCamelCase_				:






   '''simple docstring'''

   A			:  str
   A			:  str
   A			:  Optional[str]					=						None
   A			:  Optional[str]					=						None


   def 						_UpperCamelCase							(	self			):
       return json.dumps(dataclasses.asdict(self			)  ,    indent=2			) + "\n"

@dataclass(frozen=lowerCAmelCase_	)
class    lowerCamelCase_				:






   '''simple docstring'''

   A			:  List[int]
   A			:  Optional[List[int]]					=						None
   A			:  Optional[List[int]]					=						None
   A			:  Optional[Union[int, float]]					=						None


   def 						_UpperCamelCase							(	self			):
       return json.dumps(dataclasses.asdict(self			)			) + "\n"

class    lowerCamelCase_				:






   '''simple docstring'''

   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       raise NotImplementedError()


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       raise NotImplementedError()


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       raise NotImplementedError()


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       raise NotImplementedError()


   def 						_UpperCamelCase							(	self			):
       raise NotImplementedError()


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase			):
       if len(self.get_labels()			) > 1:
           UpperCamelCase__	     =      self.get_labels()[int(example.label			)]
       return example







   @classmethod
   def 						_UpperCamelCase							(	cls  ,    __UpperCamelCase  ,    __UpperCamelCase=None			):
       with open(__UpperCamelCase  ,    "r"  ,    encoding="utf-8-sig"			) as f:
           return list(csv.reader(__UpperCamelCase  ,    delimiter="\t"  ,    quotechar=__UpperCamelCase			)			)

class    lowerCamelCase_				(						lowerCAmelCase_	):






   '''simple docstring'''

   def __init__(	self  ,    __UpperCamelCase=None  ,    __UpperCamelCase=None  ,    __UpperCamelCase="classification"  ,    __UpperCamelCase=False			):
       UpperCamelCase__	     =      [] if labels is None else labels
       UpperCamelCase__	     =      [] if examples is None else examples
       UpperCamelCase__	     =      mode
       UpperCamelCase__	     =      verbose


   def __len__(	self			):
       return len(self.examples			)


   def __getitem__(	self  ,    __UpperCamelCase			):
       if isinstance(__UpperCamelCase  ,    __UpperCamelCase			):
           return SingleSentenceClassificationProcessor(labels=self.labels  ,    examples=self.examples[idx]			)
       return self.examples[idx]


   @classmethod
   def 						_UpperCamelCase							(	cls  ,    __UpperCamelCase  ,    __UpperCamelCase=""  ,    __UpperCamelCase=0  ,    __UpperCamelCase=1  ,    __UpperCamelCase=None  ,    __UpperCamelCase=False  ,    **__UpperCamelCase			):
       UpperCamelCase__	     =      cls(**__UpperCamelCase			)
       processor.add_examples_from_csv(
           __UpperCamelCase  ,    split_name=__UpperCamelCase  ,    column_label=__UpperCamelCase  ,    column_text=__UpperCamelCase  ,    column_id=__UpperCamelCase  ,    skip_first_row=__UpperCamelCase  ,    overwrite_labels=__UpperCamelCase  ,    overwrite_examples=__UpperCamelCase  ,    )
       return processor


   @classmethod
   def 						_UpperCamelCase							(	cls  ,    __UpperCamelCase  ,    __UpperCamelCase=None  ,    **__UpperCamelCase			):
       UpperCamelCase__	     =      cls(**__UpperCamelCase			)
       processor.add_examples(__UpperCamelCase  ,    labels=__UpperCamelCase			)
       return processor


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=""  ,    __UpperCamelCase=0  ,    __UpperCamelCase=1  ,    __UpperCamelCase=None  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False  ,    ):
       UpperCamelCase__	     =      self._read_tsv(__UpperCamelCase			)
       if skip_first_row:
           UpperCamelCase__	     =      lines[1:]
       UpperCamelCase__	     =      []
       UpperCamelCase__	     =      []
       UpperCamelCase__	     =      []
       for i, line in enumerate(__UpperCamelCase			):
           texts.append(line[column_text]			)
           labels.append(line[column_label]			)
           if column_id is not None:
               ids.append(line[column_id]			)
           else:
               UpperCamelCase__	     =      f'{split_name}-{i}' if split_name else str(__UpperCamelCase			)
               ids.append(__UpperCamelCase			)

       return self.add_examples(
           __UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase  ,    overwrite_labels=__UpperCamelCase  ,    overwrite_examples=__UpperCamelCase			)


   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None  ,    __UpperCamelCase=None  ,    __UpperCamelCase=False  ,    __UpperCamelCase=False			):
       if labels is not None and len(__UpperCamelCase			) != len(__UpperCamelCase			):
           raise ValueError(
               f'Text and labels have mismatched lengths {len(__UpperCamelCase			)} and {len(__UpperCamelCase			)}'			)
       if ids is not None and len(__UpperCamelCase			) != len(__UpperCamelCase			):
           raise ValueError(f'Text and ids have mismatched lengths {len(__UpperCamelCase			)} and {len(__UpperCamelCase			)}'			)
       if ids is None:
           UpperCamelCase__	     =      [None] * len(__UpperCamelCase			)
       if labels is None:
           UpperCamelCase__	     =      [None] * len(__UpperCamelCase			)
       UpperCamelCase__	     =      []
       UpperCamelCase__	     =      set()
       for text_or_text_and_label, label, guid in zip(__UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase			):
           if isinstance(__UpperCamelCase  ,    (tuple, list)			) and label is None:
               UpperCamelCase__     ,	UpperCamelCase__	     =      text_or_text_and_label
           else:
               UpperCamelCase__	     =      text_or_text_and_label
           added_labels.add(__UpperCamelCase			)
           examples.append(InputExample(guid=__UpperCamelCase  ,    text_a=__UpperCamelCase  ,    text_b=__UpperCamelCase  ,    label=__UpperCamelCase			)			)

       # Update examples
       if overwrite_examples:
           UpperCamelCase__	     =      examples
       else:
           self.examples.extend(__UpperCamelCase			)

       # Update labels
       if overwrite_labels:
           UpperCamelCase__	     =      list(__UpperCamelCase			)
       else:
           UpperCamelCase__	     =      list(set(self.labels			).union(__UpperCamelCase			)			)

       return self.examples







   def 						_UpperCamelCase							(	self  ,    __UpperCamelCase  ,    __UpperCamelCase=None  ,    __UpperCamelCase=False  ,    __UpperCamelCase=0  ,    __UpperCamelCase=True  ,    __UpperCamelCase=None  ,    ):
       if max_length is None:
           UpperCamelCase__	     =      tokenizer.max_len

       UpperCamelCase__	     =      {label: i for i, label in enumerate(self.labels			)}

       UpperCamelCase__	     =      []
       for ex_index, example in enumerate(self.examples			):
           if ex_index % 1_00_00 == 0:
               logger.info(f'Tokenizing example {ex_index}'			)

           UpperCamelCase__	     =      tokenizer.encode(
               example.text_a  ,    add_special_tokens=__UpperCamelCase  ,    max_length=min(__UpperCamelCase  ,    tokenizer.max_len			)  ,    )
           all_input_ids.append(__UpperCamelCase			)

       UpperCamelCase__	     =      max(len(__UpperCamelCase			) for input_ids in all_input_ids			)

       UpperCamelCase__	     =      []
       for ex_index, (input_ids, example) in enumerate(zip(__UpperCamelCase  ,    self.examples			)			):
           if ex_index % 1_00_00 == 0:
               logger.info(f'Writing example {ex_index}/{len(self.examples			)}'			)
           # The mask has 1 for real tokens and 0 for padding tokens. Only real
           # tokens are attended to.
           UpperCamelCase__	     =      [1 if mask_padding_with_zero else 0] * len(__UpperCamelCase			)

           # Zero-pad up to the sequence length.
           UpperCamelCase__	     =      batch_length - len(__UpperCamelCase			)
           if pad_on_left:
               UpperCamelCase__	     =      ([pad_token] * padding_length) + input_ids
               UpperCamelCase__	     =      ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
           else:
               UpperCamelCase__	     =      input_ids + ([pad_token] * padding_length)
               UpperCamelCase__	     =      attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

           if len(__UpperCamelCase			) != batch_length:
               raise ValueError(f'Error with input length {len(__UpperCamelCase			)} vs {batch_length}'			)
           if len(__UpperCamelCase			) != batch_length:
               raise ValueError(f'Error with input length {len(__UpperCamelCase			)} vs {batch_length}'			)

           if self.mode == "classification":
               UpperCamelCase__	     =      label_map[example.label]
           elif self.mode == "regression":
               UpperCamelCase__	     =      float(example.label			)
           else:
               raise ValueError(self.mode			)

           if ex_index < 5 and self.verbose:
               logger.info("*** Example ***"			)
               logger.info(f'guid: {example.guid}'			)
               logger.info(f'input_ids: {" ".join([str(__UpperCamelCase			) for x in input_ids]			)}'			)
               logger.info(f'attention_mask: {" ".join([str(__UpperCamelCase			) for x in attention_mask]			)}'			)
               logger.info(f'label: {example.label} (id = {label})'			)

           features.append(InputFeatures(input_ids=__UpperCamelCase  ,    attention_mask=__UpperCamelCase  ,    label=__UpperCamelCase			)			)

       if return_tensors is None:
           return features
       elif return_tensors == "tf":
           if not is_tf_available():
               raise RuntimeError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported"			)
           import tensorflow as tf

           def gen():
               for ex in features:
                   yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

           UpperCamelCase__	     =      tf.data.Dataset.from_generator(
               __UpperCamelCase  ,    ({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa)  ,    ({"input_ids": tf.TensorShape([None]			), "attention_mask": tf.TensorShape([None]			)}, tf.TensorShape([]			))  ,    )
           return dataset
       elif return_tensors == "pt":
           if not is_torch_available():
               raise RuntimeError("return_tensors set to 'pt' but PyTorch can't be imported"			)
           import torch
           from torch.utils.data import TensorDataset

           UpperCamelCase__	     =      torch.tensor([f.input_ids for f in features]  ,    dtype=torch.long			)
           UpperCamelCase__	     =      torch.tensor([f.attention_mask for f in features]  ,    dtype=torch.long			)
           if self.mode == "classification":
               UpperCamelCase__	     =      torch.tensor([f.label for f in features]  ,    dtype=torch.long			)
           elif self.mode == "regression":
               UpperCamelCase__	     =      torch.tensor([f.label for f in features]  ,    dtype=torch.float			)

           UpperCamelCase__	     =      TensorDataset(__UpperCamelCase  ,    __UpperCamelCase  ,    __UpperCamelCase			)
           return dataset
       else:
           raise ValueError("return_tensors should be one of 'tf' or 'pt'"			)




