def train(data_train, model, nsp_loss, mlm_loss, vocab_size, ctx, store):
    """Training function."""
    mlm_metric = nlp.metric.MaskedAccuracy()
    nsp_metric = nlp.metric.MaskedAccuracy()
    mlm_metric.reset()
    nsp_metric.reset()

    lr = args.lr
    optim_params = {'learning_rate': lr, 'epsilon': 1e-6, 'wd': 0.01}
    if args.dtype == 'float16':
        optim_params['multi_precision'] = True

    trainer = mx.gluon.Trainer(model.collect_params(), 'bertadam', optim_params,
                               update_on_kvstore=False, kvstore=store)
    dynamic_loss_scale = args.dtype == 'float16'
    fp16_trainer = FP16Trainer(trainer, dynamic_loss_scale=dynamic_loss_scale)

    if args.ckpt_dir and args.start_step:
        state_path = os.path.join(args.ckpt_dir, '%07d.states' % args.start_step)
        logging.info('Loading trainer state from %s', state_path)
        trainer.load_states(state_path)

    accumulate = args.accumulate
    num_train_steps = args.num_steps
    warmup_ratio = args.warmup_ratio
    num_warmup_steps = int(num_train_steps * warmup_ratio)
    params = [p for p in model.collect_params().values() if p.grad_req != 'null']
    param_dict = model.collect_params()

    # Do not apply weight decay on LayerNorm and bias terms
    for _, v in model.collect_params('.*beta|.*gamma|.*bias').items():
        v.wd_mult = 0.0
    if accumulate > 1:
        for p in params:
            p.grad_req = 'add'

    train_begin_time = time.time()
    begin_time = time.time()
    running_mlm_loss = running_nsp_loss = running_num_tks = 0
    batch_num = 0
    step_num = args.start_step

    parallel_model = ParallelBERT(model, mlm_loss, nsp_loss, vocab_size,
                                  store.num_workers * accumulate, trainer=fp16_trainer)
    num_ctxes = len(ctx)
    parallel = nlp.utils.Parallel(num_ctxes if num_ctxes > 1 else 0, parallel_model)

    while step_num < num_train_steps:
        for _, dataloader in enumerate(data_train):
            if step_num >= num_train_steps:
                break

            # create dummy data loader if needed
            if args.dummy_data_len:
                target_shape = (args.batch_size*num_ctxes, args.dummy_data_len)
                dataloader = get_dummy_dataloader(dataloader, target_shape)

            for _, data_batch in enumerate(dataloader):
                if step_num >= num_train_steps:
                    break
                if batch_num % accumulate == 0:
                    step_num += 1
                    # if accumulate > 1, grad_req is set to 'add', and zero_grad is required
                    if accumulate > 1:
                        param_dict.zero_grad()
                    # update learning rate
                    if step_num <= num_warmup_steps:
                        new_lr = lr * step_num / num_warmup_steps
                    else:
                        offset = lr * step_num / num_train_steps
                        new_lr = lr - offset
                    trainer.set_learning_rate(new_lr)
                    if args.profile:
                        profile(step_num, 10, 12, profile_name=args.profile)
                if args.use_avg_len:
                    data_list = [[seq.as_in_context(context) for seq in shard]
                                 for context, shard in zip(ctx, data_batch)]
                else:
                    if data_batch[0].shape[0] < len(ctx):
                        continue
                    data_list = split_and_load(data_batch, ctx)

                ns_label_list, ns_pred_list = [], []
                mask_label_list, mask_pred_list, mask_weight_list = [], [], []

                # parallel forward / backward
                for data in data_list:
                    parallel.put(data)
                for _ in range(len(ctx)):
                    (_, next_sentence_label, classified, masked_id,
                     decoded, masked_weight, ls1, ls2, valid_length) = parallel.get()
                    ns_label_list.append(next_sentence_label)
                    ns_pred_list.append(classified)
                    mask_label_list.append(masked_id)
                    mask_pred_list.append(decoded)
                    mask_weight_list.append(masked_weight)
                    running_mlm_loss += ls1.as_in_context(mx.cpu()) / num_ctxes
                    running_nsp_loss += ls2.as_in_context(mx.cpu()) / num_ctxes
                    running_num_tks += valid_length.sum().as_in_context(mx.cpu())

                # update
                if (batch_num + 1) % accumulate == 0:
                    fp16_trainer.step(1, max_norm=1)
                nsp_metric.update(ns_label_list, ns_pred_list)
                mlm_metric.update(mask_label_list, mask_pred_list, mask_weight_list)
                # logging
                if (step_num + 1) % (args.log_interval) == 0 and (batch_num + 1) % accumulate == 0:
                    log(begin_time, running_num_tks, running_mlm_loss / accumulate,
                        running_nsp_loss / accumulate, step_num, mlm_metric,
                        nsp_metric, trainer, args.log_interval)
                    begin_time = time.time()
                    running_mlm_loss = running_nsp_loss = running_num_tks = 0
                    mlm_metric.reset_local()
                    nsp_metric.reset_local()

                # saving checkpoints
                if args.ckpt_dir and (step_num + 1) % (args.ckpt_interval) == 0 \
                   and (batch_num + 1) % accumulate == 0:
                    save_params(step_num, model, trainer, args.ckpt_dir)
                batch_num += 1
    save_params(step_num, model, trainer, args.ckpt_dir)
    mx.nd.waitall()
    train_end_time = time.time()
    logging.info('Train cost={:.1f}s'.format(train_end_time - train_begin_time))
def train ( data_train , model , nsp_loss , mlm_loss , vocab_size , ctx , store ) : mlm_metric = nlp . metric . MaskedAccuracy ( ) nsp_metric = nlp . metric . MaskedAccuracy ( ) mlm_metric . reset ( ) nsp_metric . reset ( ) lr = args . lr optim_params = { 'learning_rate' : lr , 'epsilon' : 1e-6 , 'wd' : 0.01 } if args . dtype == 'float16' : optim_params [ 'multi_precision' ] = True trainer = mx . gluon . Trainer ( model . collect_params ( ) , 'bertadam' , optim_params , update_on_kvstore = False , kvstore = store ) dynamic_loss_scale = args . dtype == 'float16' fp16_trainer = FP16Trainer ( trainer , dynamic_loss_scale = dynamic_loss_scale ) if args . ckpt_dir and args . start_step : state_path = os . path . join ( args . ckpt_dir , '%07d.states' % args . start_step ) logging . info ( 'Loading trainer state from %s' , state_path ) trainer . load_states ( state_path ) accumulate = args . accumulate num_train_steps = args . num_steps warmup_ratio = args . warmup_ratio num_warmup_steps = int ( num_train_steps * warmup_ratio ) params = [ p for p in model . collect_params ( ) . values ( ) if p . grad_req != 'null' ] param_dict = model . collect_params ( ) for _ , v in model . collect_params ( '.*beta|.*gamma|.*bias' ) . items ( ) : v . wd_mult = 0.0 if accumulate > 1 : for p in params : p . grad_req = 'add' train_begin_time = time . time ( ) begin_time = time . time ( ) running_mlm_loss = running_nsp_loss = running_num_tks = 0 batch_num = 0 step_num = args . start_step parallel_model = ParallelBERT ( model , mlm_loss , nsp_loss , vocab_size , store . num_workers * accumulate , trainer = fp16_trainer ) num_ctxes = len ( ctx ) parallel = nlp . utils . Parallel ( num_ctxes if num_ctxes > 1 else 0 , parallel_model ) while step_num < num_train_steps : for _ , dataloader in enumerate ( data_train ) : if step_num >= num_train_steps : break if args . dummy_data_len : target_shape = ( args . batch_size * num_ctxes , args . dummy_data_len ) dataloader = get_dummy_dataloader ( dataloader , target_shape ) for _ , data_batch in enumerate ( dataloader ) : if step_num >= num_train_steps : break if batch_num % accumulate == 0 : step_num += 1 if accumulate > 1 : param_dict . zero_grad ( ) if step_num <= num_warmup_steps : new_lr = lr * step_num / num_warmup_steps else : offset = lr * step_num / num_train_steps new_lr = lr - offset trainer . set_learning_rate ( new_lr ) if args . profile : profile ( step_num , 10 , 12 , profile_name = args . profile ) if args . use_avg_len : data_list = [ [ seq . as_in_context ( context ) for seq in shard ] for context , shard in zip ( ctx , data_batch ) ] else : if data_batch [ 0 ] . shape [ 0 ] < len ( ctx ) : continue data_list = split_and_load ( data_batch , ctx ) ns_label_list , ns_pred_list = [ ] , [ ] mask_label_list , mask_pred_list , mask_weight_list = [ ] , [ ] , [ ] for data in data_list : parallel . put ( data ) for _ in range ( len ( ctx ) ) : ( _ , next_sentence_label , classified , masked_id , decoded , masked_weight , ls1 , ls2 , valid_length ) = parallel . get ( ) ns_label_list . append ( next_sentence_label ) ns_pred_list . append ( classified ) mask_label_list . append ( masked_id ) mask_pred_list . append ( decoded ) mask_weight_list . append ( masked_weight ) running_mlm_loss += ls1 . as_in_context ( mx . cpu ( ) ) / num_ctxes running_nsp_loss += ls2 . as_in_context ( mx . cpu ( ) ) / num_ctxes running_num_tks += valid_length . sum ( ) . as_in_context ( mx . cpu ( ) ) if ( batch_num + 1 ) % accumulate == 0 : fp16_trainer . step ( 1 , max_norm = 1 ) nsp_metric . update ( ns_label_list , ns_pred_list ) mlm_metric . update ( mask_label_list , mask_pred_list , mask_weight_list ) if ( step_num + 1 ) % ( args . log_interval ) == 0 and ( batch_num + 1 ) % accumulate == 0 : log ( begin_time , running_num_tks , running_mlm_loss / accumulate , running_nsp_loss / accumulate , step_num , mlm_metric , nsp_metric , trainer , args . log_interval ) begin_time = time . time ( ) running_mlm_loss = running_nsp_loss = running_num_tks = 0 mlm_metric . reset_local ( ) nsp_metric . reset_local ( ) if args . ckpt_dir and ( step_num + 1 ) % ( args . ckpt_interval ) == 0 and ( batch_num + 1 ) % accumulate == 0 : save_params ( step_num , model , trainer , args . ckpt_dir ) batch_num += 1 save_params ( step_num , model , trainer , args . ckpt_dir ) mx . nd . waitall ( ) train_end_time = time . time ( ) logging . info ( 'Train cost={:.1f}s' . format ( train_end_time - train_begin_time ) )
Training function.