pyspark/bigdl/dataset/news20.py
def get_news20(source_dir="./data/news20/"):
    """
    Parse or download news20 if source_dir is empty.

    :param source_dir: The directory storing news data.
    :return: A list of (tokens, label)
    """
    news_dir = download_news20(source_dir)
    texts = []  # list of text samples
    label_id = 0
    for name in sorted(os.listdir(news_dir)):
        path = os.path.join(news_dir, name)
        label_id += 1
        if os.path.isdir(path):
            for fname in sorted(os.listdir(path)):
                if fname.isdigit():
                    fpath = os.path.join(path, fname)
                    if sys.version_info < (3,):
                        f = open(fpath)
                    else:
                        f = open(fpath, encoding='latin-1')
                    content = f.read()
                    texts.append((content, label_id))
                    f.close()

    print('Found %s texts.' % len(texts))
    return texts
def get_news20(source_dir="./data/news20/"):
    news_dir = download_news20(source_dir)
    texts = []  # list of text samples
    label_id = 0
    for name in sorted(os.listdir(news_dir)):
        path = os.path.join(news_dir, name)
        label_id += 1
        if os.path.isdir(path):
            for fname in sorted(os.listdir(path)):
                if fname.isdigit():
                    fpath = os.path.join(path, fname)
                    if sys.version_info < (3,):
                        f = open(fpath)
                    else:
                        f = open(fpath, encoding='latin-1')
                    content = f.read()
                    texts.append((content, label_id))
                    f.close()

    print('Found %s texts.' % len(texts))
    return texts
def get_news20 ( source_dir = "./data/news20/" ) : news_dir = download_news20 ( source_dir ) texts = [ ] # list of text samples label_id = 0 for name in sorted ( os . listdir ( news_dir ) ) : path = os . path . join ( news_dir , name ) label_id += 1 if os . path . isdir ( path ) : for fname in sorted ( os . listdir ( path ) ) : if fname . isdigit ( ) : fpath = os . path . join ( path , fname ) if sys . version_info < ( 3 , ) : f = open ( fpath ) else : f = open ( fpath , encoding = 'latin-1' ) content = f . read ( ) texts . append ( ( content , label_id ) ) f . close ( ) print ( 'Found %s texts.' % len ( texts ) ) return texts
Parse or download news20 if source_dir is empty.

    :param source_dir: The directory storing news data.
    :return: A list of (tokens, label)
Parse or download news20 if source_dir is empty .