streamcorpus_pipeline/_spinn3r_feed_storage.py
def _make_stream_item(entry):
    """Given a single spinn3r feed entry, produce a single StreamItem.

    Returns 'None' if a complete item can't be constructed.

    """
    # get standard metadata, assuming it's present...
    if not hasattr(entry, 'permalink_entry'):
        return None
    pe = entry.permalink_entry

    # ...and create a streamitem...
    si = streamcorpus.make_stream_item(
        pe.date_found[:-1] + '.0Z',
        pe.canonical_link.href.encode('utf8'))
    if not si.stream_time:
        logger.debug('failed to generate stream_time from {0!r}'
                     .format(pe.date_found))
        return None
    if not si.abs_url:
        logger.debug('failed to generate abs_url from {0!r}'
                     .format(pe.canonical_link.href))
        return None

    # ...filling in the actual data
    si.body = _make_content_item(
        pe.content,
        alternate_data=entry.feed_entry.content.data)
    if not si.body:
        return None
    if not si.body.raw:
        return None

    if pe.content_extract.data:
        si.other_content['extract'] = _make_content_item(pe.content_extract)
    si.other_content['title'] = streamcorpus.ContentItem(
        raw=pe.title.encode('utf8'),
        media_type=pe.content_extract.mime_type,
        encoding='UTF-8')
    si.other_content['feed_entry_title'] = streamcorpus.ContentItem(
        raw=entry.feed_entry.title.encode('utf8'),
        media_type=entry.feed_entry.content.mime_type,
        encoding='UTF-8')
    if entry.feed_entry.content.data:
        si.other_content['feed_entry'] = _make_content_item(
            entry.feed_entry.content)
    si.source_metadata['lang'] = pe.lang[0].code
    si.source_metadata['author'] = json.dumps(
        dict(
            name=pe.author[0].name,
            email=pe.author[0].email,
            link=pe.author[0].link[0].href,
        )
    )
    si.source = entry.source.publisher_type
    return si
def _make_stream_item(entry):
    # get standard metadata, assuming it's present...
    if not hasattr(entry, 'permalink_entry'):
        return None
    pe = entry.permalink_entry

    # ...and create a streamitem...
    si = streamcorpus.make_stream_item(
        pe.date_found[:-1] + '.0Z',
        pe.canonical_link.href.encode('utf8'))
    if not si.stream_time:
        logger.debug('failed to generate stream_time from {0!r}'
                     .format(pe.date_found))
        return None
    if not si.abs_url:
        logger.debug('failed to generate abs_url from {0!r}'
                     .format(pe.canonical_link.href))
        return None

    # ...filling in the actual data
    si.body = _make_content_item(
        pe.content,
        alternate_data=entry.feed_entry.content.data)
    if not si.body:
        return None
    if not si.body.raw:
        return None

    if pe.content_extract.data:
        si.other_content['extract'] = _make_content_item(pe.content_extract)
    si.other_content['title'] = streamcorpus.ContentItem(
        raw=pe.title.encode('utf8'),
        media_type=pe.content_extract.mime_type,
        encoding='UTF-8')
    si.other_content['feed_entry_title'] = streamcorpus.ContentItem(
        raw=entry.feed_entry.title.encode('utf8'),
        media_type=entry.feed_entry.content.mime_type,
        encoding='UTF-8')
    if entry.feed_entry.content.data:
        si.other_content['feed_entry'] = _make_content_item(
            entry.feed_entry.content)
    si.source_metadata['lang'] = pe.lang[0].code
    si.source_metadata['author'] = json.dumps(
        dict(
            name=pe.author[0].name,
            email=pe.author[0].email,
            link=pe.author[0].link[0].href,
        )
    )
    si.source = entry.source.publisher_type
    return si
def _make_stream_item ( entry ) : # get standard metadata, assuming it's present... if not hasattr ( entry , 'permalink_entry' ) : return None pe = entry . permalink_entry # ...and create a streamitem... si = streamcorpus . make_stream_item ( pe . date_found [ : - 1 ] + '.0Z' , pe . canonical_link . href . encode ( 'utf8' ) ) if not si . stream_time : logger . debug ( 'failed to generate stream_time from {0!r}' . format ( pe . date_found ) ) return None if not si . abs_url : logger . debug ( 'failed to generate abs_url from {0!r}' . format ( pe . canonical_link . href ) ) return None # ...filling in the actual data si . body = _make_content_item ( pe . content , alternate_data = entry . feed_entry . content . data ) if not si . body : return None if not si . body . raw : return None if pe . content_extract . data : si . other_content [ 'extract' ] = _make_content_item ( pe . content_extract ) si . other_content [ 'title' ] = streamcorpus . ContentItem ( raw = pe . title . encode ( 'utf8' ) , media_type = pe . content_extract . mime_type , encoding = 'UTF-8' ) si . other_content [ 'feed_entry_title' ] = streamcorpus . ContentItem ( raw = entry . feed_entry . title . encode ( 'utf8' ) , media_type = entry . feed_entry . content . mime_type , encoding = 'UTF-8' ) if entry . feed_entry . content . data : si . other_content [ 'feed_entry' ] = _make_content_item ( entry . feed_entry . content ) si . source_metadata [ 'lang' ] = pe . lang [ 0 ] . code si . source_metadata [ 'author' ] = json . dumps ( dict ( name = pe . author [ 0 ] . name , email = pe . author [ 0 ] . email , link = pe . author [ 0 ] . link [ 0 ] . href , ) ) si . source = entry . source . publisher_type return si
Given a single spinn3r feed entry, produce a single StreamItem.

    Returns 'None' if a complete item can't be constructed.
Given a single spinn3r feed entry produce a single StreamItem .