Training process

def train():
    """Training process"""
    start_pipeline_time = time.time()

    # Training/Testing
    best_valid_acc = 0
    stop_early = 0
    for epoch in range(args.epochs):
        # Epoch training stats
        start_epoch_time = time.time()
        epoch_L = 0.0
        epoch_sent_num = 0
        epoch_wc = 0
        # Log interval training stats
        start_log_interval_time = time.time()
        log_interval_wc = 0
        log_interval_sent_num = 0
        log_interval_L = 0.0

        for i, ((data, valid_length), label) in enumerate(train_dataloader):
            data = mx.nd.transpose(data.as_in_context(context))
            label = label.as_in_context(context)
            valid_length = valid_length.as_in_context(context).astype(np.float32)
            wc = valid_length.sum().asscalar()
            log_interval_wc += wc
            epoch_wc += wc
            log_interval_sent_num += data.shape[1]
            epoch_sent_num += data.shape[1]
            with autograd.record():
                output = net(data, valid_length)
                L = loss(output, label).mean()
            L.backward()
            # Clip gradient
            if args.clip is not None:
                grads = [p.grad(context) for p in net.collect_params().values()]
                gluon.utils.clip_global_norm(grads, args.clip)
            # Update parameter
            trainer.step(1)
            log_interval_L += L.asscalar()
            epoch_L += L.asscalar()
            if (i + 1) % args.log_interval == 0:
                print('[Epoch %d Batch %d/%d] avg loss %g, throughput %gK wps' % (
                    epoch, i + 1, len(train_dataloader),
                    log_interval_L / log_interval_sent_num,
                    log_interval_wc / 1000 / (time.time() - start_log_interval_time)))
                # Clear log interval training stats
                start_log_interval_time = time.time()
                log_interval_wc = 0
                log_interval_sent_num = 0
                log_interval_L = 0
        end_epoch_time = time.time()
        valid_avg_L, valid_acc = evaluate(valid_dataloader)
        test_avg_L, test_acc = evaluate(test_dataloader)
        print('[Epoch %d] train avg loss %g, '
              'valid acc %.4f, valid avg loss %g, '
              'test acc %.4f, test avg loss %g, throughput %gK wps' % (
                  epoch, epoch_L / epoch_sent_num,
                  valid_acc, valid_avg_L, test_acc, test_avg_L,
                  epoch_wc / 1000 / (end_epoch_time - start_epoch_time)))

        if valid_acc < best_valid_acc:
            print('No Improvement.')
            stop_early += 1
            if stop_early == 3:
                break
        else:
            # Reset stop_early if the validation loss finds a new low value
            print('Observed Improvement.')
            stop_early = 0
            net.save_parameters(args.save_prefix + '_{:04d}.params'.format(epoch))
            best_valid_acc = valid_acc

    net.load_parameters(glob.glob(args.save_prefix+'_*.params')[-1], context)
    valid_avg_L, valid_acc = evaluate(valid_dataloader)
    test_avg_L, test_acc = evaluate(test_dataloader)
    print('Best validation loss %g, validation acc %.4f'%(valid_avg_L, valid_acc))
    print('Best test loss %g, test acc %.4f'%(test_avg_L, test_acc))
    print('Total time cost %.2fs'%(time.time()-start_pipeline_time))