def load_fasttext_format(cls, path, ctx=cpu(), **kwargs):
        """Create an instance of the class and load weights.

        Load the weights from the fastText binary format created by
        https://github.com/facebookresearch/fastText

        Parameters
        ----------
        path : str
            Path to the .bin model file.
        ctx : mx.Context, default mx.cpu()
            Context to initialize the weights on.
        kwargs : dict
            Keyword arguments are passed to the class initializer.

        """
        with open(path, 'rb') as f:
            new_format, dim, bucket, minn, maxn, = cls._read_model_params(f)
            idx_to_token = cls._read_vocab(f, new_format)
            dim, matrix = cls._read_vectors(f, new_format, bucket,
                                            len(idx_to_token))

        token_to_idx = {token: idx for idx, token in enumerate(idx_to_token)}
        if len(token_to_idx) != len(idx_to_token):
            # If multiple tokens with invalid encoding were collapsed in a
            # single token due to replacement of invalid bytes with Unicode
            # replacement character
            warnings.warn(
                'There are duplicate tokens in the embedding file. '
                'This is likely due to decoding errors for some tokens, '
                'where invalid bytes were replaced by '
                'the Unicode replacement character. '
                'This affects {} tokens.'.format(
                    len(idx_to_token) - len(token_to_idx)))
            for _ in range(len(token_to_idx), len(idx_to_token)):
                # Add pseudo tokens to make sure length is the same
                token_to_idx[object()] = -1
        assert len(token_to_idx) == len(idx_to_token)

        subword_function = create_subword_function(
            'NGramHashes', num_subwords=matrix.shape[0] - len(idx_to_token),
            ngrams=list(range(minn, maxn + 1)), special_tokens={'</s>'})

        self = cls(token_to_idx, subword_function, output_dim=dim, **kwargs)

        self.initialize(ctx=ctx)
        self.weight.set_data(nd.array(matrix))

        return self
def load_fasttext_format ( cls , path , ctx = cpu ( ) , ** kwargs ) : with open ( path , 'rb' ) as f : new_format , dim , bucket , minn , maxn , = cls . _read_model_params ( f ) idx_to_token = cls . _read_vocab ( f , new_format ) dim , matrix = cls . _read_vectors ( f , new_format , bucket , len ( idx_to_token ) ) token_to_idx = { token : idx for idx , token in enumerate ( idx_to_token ) } if len ( token_to_idx ) != len ( idx_to_token ) : warnings . warn ( 'There are duplicate tokens in the embedding file. ' 'This is likely due to decoding errors for some tokens, ' 'where invalid bytes were replaced by ' 'the Unicode replacement character. ' 'This affects {} tokens.' . format ( len ( idx_to_token ) - len ( token_to_idx ) ) ) for _ in range ( len ( token_to_idx ) , len ( idx_to_token ) ) : token_to_idx [ object ( ) ] = - 1 assert len ( token_to_idx ) == len ( idx_to_token ) subword_function = create_subword_function ( 'NGramHashes' , num_subwords = matrix . shape [ 0 ] - len ( idx_to_token ) , ngrams = list ( range ( minn , maxn + 1 ) ) , special_tokens = { '</s>' } ) self = cls ( token_to_idx , subword_function , output_dim = dim , ** kwargs ) self . initialize ( ctx = ctx ) self . weight . set_data ( nd . array ( matrix ) ) return self
Create an instance of the class and load weights.

        Load the weights from the fastText binary format created by
        https://github.com/facebookresearch/fastText

        Parameters
        ----------
        path : str
            Path to the .bin model file.
        ctx : mx.Context, default mx.cpu()
            Context to initialize the weights on.
        kwargs : dict
            Keyword arguments are passed to the class initializer.