def _load_embedding_txt(self, pretrained_file_path, elem_delim, encoding='utf8'):
        """Load embedding vectors from a pre-trained token embedding file.

        For every unknown token, if its representation `self.unknown_token` is encountered in the
        pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token
        embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the
        text embedding vector initialized by `self._init_unknown_vec`.

        If a token is encountered multiple times in the pre-trained text embedding file, only the
        first-encountered token embedding vector will be loaded and the rest will be skipped.
        """

        vec_len = None
        all_elems = []
        tokens = set()
        loaded_unknown_vec = None
        with io.open(pretrained_file_path, 'rb') as f:
            for line_num, line in enumerate(f):
                try:
                    line = line.decode(encoding)
                except ValueError:
                    warnings.warn('line {} in {}: failed to decode. Skipping.'
                                  .format(line_num, pretrained_file_path))
                    continue

                elems = line.rstrip().split(elem_delim)

                assert len(elems) > 1, 'line {} in {}: unexpected data format.'.format(
                    line_num, pretrained_file_path)

                token, elems = elems[0], [float(i) for i in elems[1:]]

                if token == self.unknown_token and loaded_unknown_vec is None:
                    loaded_unknown_vec = elems
                    tokens.add(self.unknown_token)
                elif token in tokens:
                    warnings.warn('line {} in {}: duplicate embedding found for '
                                  'token "{}". Skipped.'.format(line_num, pretrained_file_path,
                                                                token))
                elif len(elems) == 1 and line_num == 0:
                    warnings.warn('line {} in {}: skipped likely header line.'
                                  .format(line_num, pretrained_file_path))
                else:
                    if not vec_len:
                        vec_len = len(elems)
                        if self.unknown_token:
                            # Reserve a vector slot for the unknown token at the very beggining
                            # because the unknown token index is 0.
                            all_elems.extend([0] * vec_len)
                    else:
                        assert len(elems) == vec_len, \
                            'line {} in {}: found vector of inconsistent dimension for token ' \
                            '"{}". expected dim: {}, found: {}'.format(line_num,
                                                                       pretrained_file_path,
                                                                       token, vec_len, len(elems))
                    all_elems.extend(elems)
                    self._idx_to_token.append(token)
                    self._token_to_idx[token] = len(self._idx_to_token) - 1
                    tokens.add(token)

        self._idx_to_vec = nd.array(all_elems).reshape((-1, vec_len))

        if self.unknown_token:
            if loaded_unknown_vec is None:
                self._idx_to_vec[C.UNK_IDX] = self._init_unknown_vec(shape=vec_len)
            else:
                self._idx_to_vec[C.UNK_IDX] = nd.array(loaded_unknown_vec)
def _load_embedding_txt ( self , pretrained_file_path , elem_delim , encoding = 'utf8' ) : vec_len = None all_elems = [ ] tokens = set ( ) loaded_unknown_vec = None with io . open ( pretrained_file_path , 'rb' ) as f : for line_num , line in enumerate ( f ) : try : line = line . decode ( encoding ) except ValueError : warnings . warn ( 'line {} in {}: failed to decode. Skipping.' . format ( line_num , pretrained_file_path ) ) continue elems = line . rstrip ( ) . split ( elem_delim ) assert len ( elems ) > 1 , 'line {} in {}: unexpected data format.' . format ( line_num , pretrained_file_path ) token , elems = elems [ 0 ] , [ float ( i ) for i in elems [ 1 : ] ] if token == self . unknown_token and loaded_unknown_vec is None : loaded_unknown_vec = elems tokens . add ( self . unknown_token ) elif token in tokens : warnings . warn ( 'line {} in {}: duplicate embedding found for ' 'token "{}". Skipped.' . format ( line_num , pretrained_file_path , token ) ) elif len ( elems ) == 1 and line_num == 0 : warnings . warn ( 'line {} in {}: skipped likely header line.' . format ( line_num , pretrained_file_path ) ) else : if not vec_len : vec_len = len ( elems ) if self . unknown_token : all_elems . extend ( [ 0 ] * vec_len ) else : assert len ( elems ) == vec_len , 'line {} in {}: found vector of inconsistent dimension for token ' '"{}". expected dim: {}, found: {}' . format ( line_num , pretrained_file_path , token , vec_len , len ( elems ) ) all_elems . extend ( elems ) self . _idx_to_token . append ( token ) self . _token_to_idx [ token ] = len ( self . _idx_to_token ) - 1 tokens . add ( token ) self . _idx_to_vec = nd . array ( all_elems ) . reshape ( ( - 1 , vec_len ) ) if self . unknown_token : if loaded_unknown_vec is None : self . _idx_to_vec [ C . UNK_IDX ] = self . _init_unknown_vec ( shape = vec_len ) else : self . _idx_to_vec [ C . UNK_IDX ] = nd . array ( loaded_unknown_vec )
Load embedding vectors from a pre-trained token embedding file.

        For every unknown token, if its representation `self.unknown_token` is encountered in the
        pre-trained token embedding file, index 0 of `self.idx_to_vec` maps to the pre-trained token
        embedding vector loaded from the file; otherwise, index 0 of `self.idx_to_vec` maps to the
        text embedding vector initialized by `self._init_unknown_vec`.

        If a token is encountered multiple times in the pre-trained text embedding file, only the
        first-encountered token embedding vector will be loaded and the rest will be skipped.