def __call__(self, line):
<EOL>text_a = line[<NUM_LIT:0>]<EOL>if self._pair:<EOL><INDENT>assert len(line) == <NUM_LIT:2><EOL>text_b = line[<NUM_LIT:1>]<EOL><DEDENT>tokens_a = self._tokenizer(text_a)<EOL>tokens_b = None<EOL>if self._pair:<EOL><INDENT>tokens_b = self._tokenizer(text_b)<EOL><DEDENT>if tokens_b:<EOL><INDENT>self._truncate_seq_pair(tokens_a, tokens_b, self._max_seq_length - <NUM_LIT:3>)<EOL><DEDENT>else:<EOL><INDENT>if len(tokens_a) > self._max_seq_length - <NUM_LIT:2>:<EOL><INDENT>tokens_a = tokens_a[<NUM_LIT:0>:(self._max_seq_length - <NUM_LIT:2>)]<EOL><DEDENT><DEDENT>vocab = self._tokenizer.vocab<EOL>tokens = []<EOL>tokens.append(vocab.cls_token)<EOL>tokens.extend(tokens_a)<EOL>tokens.append(vocab.sep_token)<EOL>segment_ids = [<NUM_LIT:0>] * len(tokens)<EOL>if tokens_b:<EOL><INDENT>tokens.extend(tokens_b)<EOL>tokens.append(vocab.sep_token)<EOL>segment_ids.extend([<NUM_LIT:1>] * (len(tokens) - len(segment_ids)))<EOL><DEDENT>input_ids = self._tokenizer.convert_tokens_to_ids(tokens)<EOL>valid_length = len(input_ids)<EOL>if self._pad:<EOL><INDENT>padding_length = self._max_seq_length - valid_length<EOL>input_ids.extend([vocab[vocab.padding_token]] * padding_length)<EOL>segment_ids.extend([<NUM_LIT:0>] * padding_length)<EOL><DEDENT>return np.array(input_ids, dtype='<STR_LIT>'), np.array(valid_length, dtype='<STR_LIT>'),np.array(segment_ids, dtype='<STR_LIT>')<EOL>
Perform transformation for sequence pairs or single sequences.

        The transformation is processed in the following steps:
        - tokenize the input sequences
        - insert [CLS], [SEP] as necessary
        - generate type ids to indicate whether a token belongs to the first
        sequence or the second sequence.
        - generate valid length

        For sequence pairs, the input is a tuple of 2 strings:
        text_a, text_b.

        Inputs:
            text_a: 'is this jacksonville ?'
            text_b: 'no it is not'
        Tokenization:
            text_a: 'is this jack ##son ##ville ?'
            text_b: 'no it is not .'
        Processed:
            tokens: '[CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]'
            type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
            valid_length: 14

        For single sequences, the input is a tuple of single string:
        text_a.

        Inputs:
            text_a: 'the dog is hairy .'
        Tokenization:
            text_a: 'the dog is hairy .'
        Processed:
            text_a: '[CLS] the dog is hairy . [SEP]'
            type_ids: 0     0   0   0  0     0 0
            valid_length: 7

        Parameters
        ----------
        line: tuple of str
            Input strings. For sequence pairs, the input is a tuple of 2 strings:
            (text_a, text_b). For single sequences, the input is a tuple of single
            string: (text_a,).

        Returns
        -------
        np.array: input token ids in 'int32', shape (batch_size, seq_length)
        np.array: valid length in 'int32', shape (batch_size,)
        np.array: input token type ids in 'int32', shape (batch_size, seq_length)