

def harvest(self, start_date=None, end_date=None):
    start_date = (start_date or (datetime.date.today() - datetime.timedelta(settings.DAYS_BACK)))
    end_date = (end_date or datetime.date.today())
    shas = fetch_commits(self.BASE_URL, start_date.isoformat(), end_date.isoformat())
    files = list(set(chain.from_iterable([fetch_file_names(self.BASE_COMMIT_URL, sha) for sha in shas])))
    files = filter((lambda filename: filename.endswith('.xml')), files)
    xml_records = [fetch_xml(self.BASE_DATA_URL, filename) for filename in files]
    return [RawDocument({
        'filetype': 'xml',
        'source': self.short_name,
        'doc': etree.tostring(record),
        'docID': record.xpath('//article-id[@*]')[0].text,
    }) for record in xml_records]

dataset/ETHPy150Open CenterForOpenScience/scrapi/scrapi/harvesters/elife.py ELifeHarvester.harvest/original