Crawl and extract articles form the news crawl provided by commoncrawl.org. For each article that was extracted
    successfully the callback function callback_on_article_extracted is invoked where the first parameter is the
    article object.
    :param continue_process:
    :param delete_warc_after_extraction:
    :param number_of_extraction_processes:
    :param callback_on_article_extracted:
    :param valid_hosts:
    :param start_date:
    :param end_date:
    :param strict_date:
    :param reuse_previously_downloaded_files:
    :param local_download_dir_warc:
    :param continue_after_error:
    :param show_download_progress:
    :param log_level:
    :return:

def crawl_from_commoncrawl(callback_on_article_extracted, valid_hosts=None, start_date=None, end_date=None,
                           strict_date=True, reuse_previously_downloaded_files=True, local_download_dir_warc=None,
                           continue_after_error=True, show_download_progress=False,
                           number_of_extraction_processes=4, log_level=logging.ERROR,
                           delete_warc_after_extraction=True, continue_process=True):
    """
    Crawl and extract articles form the news crawl provided by commoncrawl.org. For each article that was extracted
    successfully the callback function callback_on_article_extracted is invoked where the first parameter is the
    article object.
    :param continue_process:
    :param delete_warc_after_extraction:
    :param number_of_extraction_processes:
    :param callback_on_article_extracted:
    :param valid_hosts:
    :param start_date:
    :param end_date:
    :param strict_date:
    :param reuse_previously_downloaded_files:
    :param local_download_dir_warc:
    :param continue_after_error:
    :param show_download_progress:
    :param log_level:
    :return:
    """
    __setup(local_download_dir_warc, log_level)

    cc_news_crawl_names = __get_remote_index()
    __logger.info('found %i files at commoncrawl.org', len(cc_news_crawl_names))

    # multiprocessing (iterate the list of crawl_names, and for each: download and process it)
    __logger.info('creating extraction process pool with %i processes', number_of_extraction_processes)
    warc_download_urls = []
    fully_extracted_warc_urls = __get_list_of_fully_extracted_warc_urls()
    for name in cc_news_crawl_names:
        warc_download_url = __get_download_url(name)
        if continue_process:
            # check if the current WARC has already been fully extracted (assuming that the filter criteria have not
            # been changed!)
            if warc_download_url in fully_extracted_warc_urls:
                __logger.info('skipping WARC because fully extracted: %s' % warc_download_url)
                pass
            else:
                warc_download_urls.append(warc_download_url)

        else:
            # if not continue process, then always add
            warc_download_urls.append(warc_download_url)

    # run the crawler in the current, single process if number of extraction processes is set to 1
    if number_of_extraction_processes > 1:
        with Pool(number_of_extraction_processes) as extraction_process_pool:
            extraction_process_pool.map(partial(__start_commoncrawl_extractor,
                                                callback_on_article_extracted=callback_on_article_extracted,
                                                valid_hosts=valid_hosts,
                                                start_date=start_date, end_date=end_date,
                                                strict_date=strict_date,
                                                reuse_previously_downloaded_files=reuse_previously_downloaded_files,
                                                local_download_dir_warc=local_download_dir_warc,
                                                continue_after_error=continue_after_error,
                                                show_download_progress=show_download_progress,
                                                log_level=log_level,
                                                delete_warc_after_extraction=delete_warc_after_extraction,
                                                log_pathname_fully_extracted_warcs=__log_pathname_fully_extracted_warcs),
                                        warc_download_urls)
    else:
        for warc_download_url in warc_download_urls:
            __start_commoncrawl_extractor(warc_download_url,
                                          callback_on_article_extracted=callback_on_article_extracted,
                                          valid_hosts=valid_hosts,
                                          start_date=start_date, end_date=end_date,
                                          strict_date=strict_date,
                                          reuse_previously_downloaded_files=reuse_previously_downloaded_files,
                                          local_download_dir_warc=local_download_dir_warc,
                                          continue_after_error=continue_after_error,
                                          show_download_progress=show_download_progress,
                                          log_level=log_level,
                                          delete_warc_after_extraction=delete_warc_after_extraction,
                                          log_pathname_fully_extracted_warcs=__log_pathname_fully_extracted_warcs)