:modelfunc: is a function that takes a word and returns its
    splits.  for ngram model this function returns all the ngrams of a
    word, for PCFG it will return splits of the password.
    @modelfunc: func: string -> [list of strings]
    @fname: name of the file to read from
    @listw: list of passwords. Used passwords from both the files and
            listw if provided.
    @outfname: the file to write down the model.

def create_model(modelfunc, fname='', listw=[], outfname='',
                 limit=int(3e6), min_pwlen=6, topk=10000, sep=r'\s+'):
    """:modelfunc: is a function that takes a word and returns its
    splits.  for ngram model this function returns all the ngrams of a
    word, for PCFG it will return splits of the password.
    @modelfunc: func: string -> [list of strings]
    @fname: name of the file to read from
    @listw: list of passwords. Used passwords from both the files and
            listw if provided.
    @outfname: the file to write down the model.
    """

    def length_filter(pw):
        pw = ''.join(c for c in pw if c in VALID_CHARS)
        return len(pw) >= min_pwlen

    pws = []
    if fname:
        pws = helper.open_get_line(fname, limit=limit, pw_filter=length_filter, sep=sep)

    big_dict = defaultdict(int)
    total_f, total_e = 0, 0
    # Add topk passwords from the input dataset to the list
    topk_pws = []
    for pw, c in itertools.chain(pws, listw):
        for ng in modelfunc(pw):
            big_dict[ng] += c
        total_f += c
        total_e += 1
        if len(big_dict) % 100000 == 0:
            print(("Dictionary size: {} (Total_freq: {}; Total_pws: {}"\
                   .format(len(big_dict), total_f, total_e)))
        if len(topk_pws) >= topk:
            heapq.heappushpop(topk_pws, (c, pw))
        else:
            heapq.heappush(topk_pws, (c, pw))
    # Adding topk password to deal with probability reduction of popular
    # passwords. Mostly effective for n-gram models
    print("topk={}".format(topk))
    if topk > 0:
        for c, pw in topk_pws:
            tpw = helper.START + pw + helper.END
            big_dict[tpw] += c
            total_f += c
            total_e += 1

    big_dict[NPWS_W] = total_e
    big_dict[TOTALF_W] = total_f

    nDawg = dawg.IntCompletionDAWG(big_dict)
    if not outfname:
        outfname = 'tmpmodel.dawg.gz'
    elif not outfname.endswith('.gz'):
        outfname += '.gz'
    pathlib.Path(outfname).parent.mkdir(parents=True, exist_ok=True)
    helper.save_dawg(nDawg, outfname)
    return nDawg