def pretrain(
    texts_loc,
    vectors_model,
    output_dir,
    width=96,
    depth=4,
    embed_rows=2000,
    loss_func="cosine",
    use_vectors=False,
    dropout=0.2,
    n_iter=1000,
    batch_size=3000,
    max_length=500,
    min_length=5,
    seed=0,
    n_save_every=None,
):
    """
    Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components,
    using an approximate language-modelling objective. Specifically, we load
    pre-trained vectors, and train a component like a CNN, BiLSTM, etc to predict
    vectors which match the pre-trained ones. The weights are saved to a directory
    after each epoch. You can then pass a path to one of these pre-trained weights
    files to the 'spacy train' command.

    This technique may be especially helpful if you have little labelled data.
    However, it's still quite experimental, so your mileage may vary.

    To load the weights back in during 'spacy train', you need to ensure
    all settings are the same between pretraining and training. The API and
    errors around this need some improvement.
    """
    config = dict(locals())
    msg = Printer()
    util.fix_random_seed(seed)

    has_gpu = prefer_gpu()
    msg.info("Using GPU" if has_gpu else "Not using GPU")

    output_dir = Path(output_dir)
    if not output_dir.exists():
        output_dir.mkdir()
        msg.good("Created output directory")
    srsly.write_json(output_dir / "config.json", config)
    msg.good("Saved settings to config.json")

    # Load texts from file or stdin
    if texts_loc != "-":  # reading from a file
        texts_loc = Path(texts_loc)
        if not texts_loc.exists():
            msg.fail("Input text file doesn't exist", texts_loc, exits=1)
        with msg.loading("Loading input texts..."):
            texts = list(srsly.read_jsonl(texts_loc))
        msg.good("Loaded input texts")
        random.shuffle(texts)
    else:  # reading from stdin
        msg.text("Reading input text from stdin...")
        texts = srsly.read_jsonl("-")

    with msg.loading("Loading model '{}'...".format(vectors_model)):
        nlp = util.load_model(vectors_model)
    msg.good("Loaded model '{}'".format(vectors_model))
    pretrained_vectors = None if not use_vectors else nlp.vocab.vectors.name
    model = create_pretraining_model(
        nlp,
        Tok2Vec(
            width,
            embed_rows,
            conv_depth=depth,
            pretrained_vectors=pretrained_vectors,
            bilstm_depth=0,  # Requires PyTorch. Experimental.
            cnn_maxout_pieces=3,  # You can try setting this higher
            subword_features=True,  # Set to False for Chinese etc
        ),
    )
    optimizer = create_default_optimizer(model.ops)
    tracker = ProgressTracker(frequency=10000)
    msg.divider("Pre-training tok2vec layer")
    row_settings = {"widths": (3, 10, 10, 6, 4), "aligns": ("r", "r", "r", "r", "r")}
    msg.row(("#", "# Words", "Total Loss", "Loss", "w/s"), **row_settings)

    def _save_model(epoch, is_temp=False):
        is_temp_str = ".temp" if is_temp else ""
        with model.use_params(optimizer.averages):
            with (output_dir / ("model%d%s.bin" % (epoch, is_temp_str))).open(
                "wb"
            ) as file_:
                file_.write(model.tok2vec.to_bytes())
            log = {
                "nr_word": tracker.nr_word,
                "loss": tracker.loss,
                "epoch_loss": tracker.epoch_loss,
                "epoch": epoch,
            }
            with (output_dir / "log.jsonl").open("a") as file_:
                file_.write(srsly.json_dumps(log) + "\n")

    for epoch in range(n_iter):
        for batch_id, batch in enumerate(
            util.minibatch_by_words(((text, None) for text in texts), size=batch_size)
        ):
            docs = make_docs(
                nlp,
                [text for (text, _) in batch],
                max_length=max_length,
                min_length=min_length,
            )
            loss = make_update(
                model, docs, optimizer, objective=loss_func, drop=dropout
            )
            progress = tracker.update(epoch, loss, docs)
            if progress:
                msg.row(progress, **row_settings)
                if texts_loc == "-" and tracker.words_per_epoch[epoch] >= 10 ** 7:
                    break
            if n_save_every and (batch_id % n_save_every == 0):
                _save_model(epoch, is_temp=True)
        _save_model(epoch)
        tracker.epoch_loss = 0.0
        if texts_loc != "-":
            # Reshuffle the texts if texts were loaded from a file
            random.shuffle(texts)
def pretrain ( texts_loc , vectors_model , output_dir , width = 96 , depth = 4 , embed_rows = 2000 , loss_func = "cosine" , use_vectors = False , dropout = 0.2 , n_iter = 1000 , batch_size = 3000 , max_length = 500 , min_length = 5 , seed = 0 , n_save_every = None , ) : config = dict ( locals ( ) ) msg = Printer ( ) util . fix_random_seed ( seed ) has_gpu = prefer_gpu ( ) msg . info ( "Using GPU" if has_gpu else "Not using GPU" ) output_dir = Path ( output_dir ) if not output_dir . exists ( ) : output_dir . mkdir ( ) msg . good ( "Created output directory" ) srsly . write_json ( output_dir / "config.json" , config ) msg . good ( "Saved settings to config.json" ) if texts_loc != "-" : texts_loc = Path ( texts_loc ) if not texts_loc . exists ( ) : msg . fail ( "Input text file doesn't exist" , texts_loc , exits = 1 ) with msg . loading ( "Loading input texts..." ) : texts = list ( srsly . read_jsonl ( texts_loc ) ) msg . good ( "Loaded input texts" ) random . shuffle ( texts ) else : msg . text ( "Reading input text from stdin..." ) texts = srsly . read_jsonl ( "-" ) with msg . loading ( "Loading model '{}'..." . format ( vectors_model ) ) : nlp = util . load_model ( vectors_model ) msg . good ( "Loaded model '{}'" . format ( vectors_model ) ) pretrained_vectors = None if not use_vectors else nlp . vocab . vectors . name model = create_pretraining_model ( nlp , Tok2Vec ( width , embed_rows , conv_depth = depth , pretrained_vectors = pretrained_vectors , bilstm_depth = 0 , cnn_maxout_pieces = 3 , subword_features = True , ) , ) optimizer = create_default_optimizer ( model . ops ) tracker = ProgressTracker ( frequency = 10000 ) msg . divider ( "Pre-training tok2vec layer" ) row_settings = { "widths" : ( 3 , 10 , 10 , 6 , 4 ) , "aligns" : ( "r" , "r" , "r" , "r" , "r" ) } msg . row ( ( "#" , "# Words" , "Total Loss" , "Loss" , "w/s" ) , ** row_settings ) def _save_model ( epoch , is_temp = False ) : is_temp_str = ".temp" if is_temp else "" with model . use_params ( optimizer . averages ) : with ( output_dir / ( "model%d%s.bin" % ( epoch , is_temp_str ) ) ) . open ( "wb" ) as file_ : file_ . write ( model . tok2vec . to_bytes ( ) ) log = { "nr_word" : tracker . nr_word , "loss" : tracker . loss , "epoch_loss" : tracker . epoch_loss , "epoch" : epoch , } with ( output_dir / "log.jsonl" ) . open ( "a" ) as file_ : file_ . write ( srsly . json_dumps ( log ) + "\n" ) for epoch in range ( n_iter ) : for batch_id , batch in enumerate ( util . minibatch_by_words ( ( ( text , None ) for text in texts ) , size = batch_size ) ) : docs = make_docs ( nlp , [ text for ( text , _ ) in batch ] , max_length = max_length , min_length = min_length , ) loss = make_update ( model , docs , optimizer , objective = loss_func , drop = dropout ) progress = tracker . update ( epoch , loss , docs ) if progress : msg . row ( progress , ** row_settings ) if texts_loc == "-" and tracker . words_per_epoch [ epoch ] >= 10 ** 7 : break if n_save_every and ( batch_id % n_save_every == 0 ) : _save_model ( epoch , is_temp = True ) _save_model ( epoch ) tracker . epoch_loss = 0.0 if texts_loc != "-" : random . shuffle ( texts )
Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components,
    using an approximate language-modelling objective. Specifically, we load
    pre-trained vectors, and train a component like a CNN, BiLSTM, etc to predict
    vectors which match the pre-trained ones. The weights are saved to a directory
    after each epoch. You can then pass a path to one of these pre-trained weights
    files to the 'spacy train' command.

    This technique may be especially helpful if you have little labelled data.
    However, it's still quite experimental, so your mileage may vary.

    To load the weights back in during 'spacy train', you need to ensure
    all settings are the same between pretraining and training. The API and
    errors around this need some improvement.