def preprocess_data(tokenizer, task, batch_size, dev_batch_size, max_len, pad=False):
<EOL>trans = BERTDatasetTransform(<EOL>tokenizer,<EOL>max_len,<EOL>labels=task.get_labels(),<EOL>pad=pad,<EOL>pair=task.is_pair,<EOL>label_dtype='<STR_LIT>' if not task.get_labels() else '<STR_LIT>')<EOL>data_train = task('<STR_LIT:train>').transform(trans, lazy=False)<EOL>data_train_len = data_train.transform(<EOL>lambda input_id, length, segment_id, label_id: length)<EOL>num_samples_train = len(data_train)<EOL>batchify_fn = nlp.data.batchify.Tuple(<EOL>nlp.data.batchify.Pad(axis=<NUM_LIT:0>), nlp.data.batchify.Stack(),<EOL>nlp.data.batchify.Pad(axis=<NUM_LIT:0>),<EOL>nlp.data.batchify.Stack(<EOL>'<STR_LIT>' if not task.get_labels() else '<STR_LIT>'))<EOL>batch_sampler = nlp.data.sampler.FixedBucketSampler(<EOL>data_train_len,<EOL>batch_size=batch_size,<EOL>num_buckets=<NUM_LIT:10>,<EOL>ratio=<NUM_LIT:0>,<EOL>shuffle=True)<EOL>dataloader_train = gluon.data.DataLoader(<EOL>dataset=data_train,<EOL>num_workers=<NUM_LIT:1>,<EOL>batch_sampler=batch_sampler,<EOL>batchify_fn=batchify_fn)<EOL>if task.task_name == '<STR_LIT>':<EOL><INDENT>data_dev_matched = task('<STR_LIT>').transform(trans, lazy=False)<EOL>data_dev_mismatched = task('<STR_LIT>').transform(trans, lazy=False)<EOL>dataloader_dev_matched = mx.gluon.data.DataLoader(<EOL>data_dev_matched, batch_size=dev_batch_size,<EOL>num_workers=<NUM_LIT:1>, shuffle=False, batchify_fn=batchify_fn)<EOL>dataloader_dev_mismatched = mx.gluon.data.DataLoader(<EOL>data_dev_mismatched, batch_size=dev_batch_size,<EOL>num_workers=<NUM_LIT:1>, shuffle=False, batchify_fn=batchify_fn)<EOL>return dataloader_train, dataloader_dev_matched,dataloader_dev_mismatched, num_samples_train<EOL><DEDENT>else:<EOL><INDENT>data_dev = task('<STR_LIT>').transform(trans, lazy=False)<EOL>dataloader_dev = mx.gluon.data.DataLoader(<EOL>data_dev,<EOL>batch_size=dev_batch_size,<EOL>num_workers=<NUM_LIT:1>,<EOL>shuffle=False,<EOL>batchify_fn=batchify_fn)<EOL>return dataloader_train, dataloader_dev, num_samples_train<EOL><DEDENT>
Data preparation function.