def main():
    """Keywords generator command line"""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('learn', help="learning source codes directory")
    parser.add_argument('keywords', help="output keywords file, JSON")
    parser.add_argument(
        '-n', '--nbkeywords', type=int, default=10000,
        help="the number of keywords to keep")
    parser.add_argument(
        '-d', '--debug', default=False, action='store_true',
        help="show debug messages")

    args = parser.parse_args()
    config_logging(args.debug)

    learn_path = Path(args.learn)
    keywords_path = Path(args.keywords)
    nb_keywords = args.nbkeywords

    languages = config_dict('languages.json')
    exts = {ext: lang for lang, exts in languages.items() for ext in exts}

    term_count = Counter()
    document_count = Counter()
    pos = 0
    LOGGER.info("Reading files form %s", learn_path)
    for pos, path in enumerate(Path(learn_path).glob('**/*'), 1):
        if pos % STEP == 0:
            LOGGER.debug("Processed %d", pos)
            gc.collect()  # Cleanup dirt

        if not path.is_file() or not exts.get(path.suffix.lstrip('.')):
            continue

        counter = _extract(path)
        term_count.update(counter)
        document_count.update(counter.keys())

    nb_terms = sum(term_count.values())
    nb_documents = pos - 1
    if not nb_documents:
        LOGGER.error("No source files found in %s", learn_path)
        raise RuntimeError('No source files in {}'.format(learn_path))

    LOGGER.info("%d unique terms found", len(term_count))

    terms = _most_frequent(
        (term_count, nb_terms), (document_count, nb_documents), nb_keywords)

    keywords = {
        token: int(hashlib.sha1(token.encode()).hexdigest(), 16)
        for token in terms
    }

    with keywords_path.open('w') as keywords_file:
        json.dump(keywords, keywords_file, indent=2, sort_keys=True)
    LOGGER.info("%d keywords written into %s", len(keywords), keywords_path)
    LOGGER.debug("Exit OK")
def main ( ) : parser = argparse . ArgumentParser ( description = __doc__ ) parser . add_argument ( 'learn' , help = "learning source codes directory" ) parser . add_argument ( 'keywords' , help = "output keywords file, JSON" ) parser . add_argument ( '-n' , '--nbkeywords' , type = int , default = 10000 , help = "the number of keywords to keep" ) parser . add_argument ( '-d' , '--debug' , default = False , action = 'store_true' , help = "show debug messages" ) args = parser . parse_args ( ) config_logging ( args . debug ) learn_path = Path ( args . learn ) keywords_path = Path ( args . keywords ) nb_keywords = args . nbkeywords languages = config_dict ( 'languages.json' ) exts = { ext : lang for lang , exts in languages . items ( ) for ext in exts } term_count = Counter ( ) document_count = Counter ( ) pos = 0 LOGGER . info ( "Reading files form %s" , learn_path ) for pos , path in enumerate ( Path ( learn_path ) . glob ( '**/*' ) , 1 ) : if pos % STEP == 0 : LOGGER . debug ( "Processed %d" , pos ) gc . collect ( ) if not path . is_file ( ) or not exts . get ( path . suffix . lstrip ( '.' ) ) : continue counter = _extract ( path ) term_count . update ( counter ) document_count . update ( counter . keys ( ) ) nb_terms = sum ( term_count . values ( ) ) nb_documents = pos - 1 if not nb_documents : LOGGER . error ( "No source files found in %s" , learn_path ) raise RuntimeError ( 'No source files in {}' . format ( learn_path ) ) LOGGER . info ( "%d unique terms found" , len ( term_count ) ) terms = _most_frequent ( ( term_count , nb_terms ) , ( document_count , nb_documents ) , nb_keywords ) keywords = { token : int ( hashlib . sha1 ( token . encode ( ) ) . hexdigest ( ) , 16 ) for token in terms } with keywords_path . open ( 'w' ) as keywords_file : json . dump ( keywords , keywords_file , indent = 2 , sort_keys = True ) LOGGER . info ( "%d keywords written into %s" , len ( keywords ) , keywords_path ) LOGGER . debug ( "Exit OK" )
Keywords generator command line