<gh_stars>10-100
#!/usr/bin/env python3
# See: https://github.com/VirusTrack/COVIDvu/blob/master/LICENSE 
# vim: set fileencoding=utf-8:


from bs4 import BeautifulSoup

from covidvu.pipeline.vujson import SITE_DATA
from covidvu.pipeline.vuupdate import SCRAPED_US_DATA
from covidvu.pipeline.vuupdate import SCRAPED_WORLD_DATA

import copy
import csv
import os


# --- constants ---

TABLES_FILES     = {
                    'LOCATION': { 
                        'fileCSV' : SCRAPED_WORLD_DATA, 
                        'fileHTML': None,
                        'ignoreRows': 6,
                        'tableHTML' : None,
                    },
                    'UNITED STATES': { 
                        'fileCSV' : SCRAPED_US_DATA, 
                        'fileHTML': None,
                        'ignoreRows': 5,
                        'tableHTML' : None,
                    },
                    'AUSTRALIA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'MAINLAND CHINA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'CANADA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'MUNDO HISPANO': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                   }


# +++ functions +++

def detectHTMLTablesRegions(dataLake = SITE_DATA):
    """
    Returns a dictionary of file names like:

    {
        'Global': 'table-00.html',
        'US'    " 'table-01.html',
    }
    """
    prefix      = 'table-'
    filesList   = [ os.path.join(dataLake, fileName) for fileName in os.listdir(dataLake) if prefix in fileName and '.swp' not in fileName ]
    tablesFiles = copy.deepcopy(TABLES_FILES)

    for fileName in filesList:
        table = BeautifulSoup(open(fileName).read(), 'html.parser')

        for row in table.find_all('tr'):
            column = row.find('td')
            if not column:
                continue
            
            text = column.text.strip().upper()

            if text in tablesFiles:
                tablesFiles[text]['fileHTML'] = fileName
                break

    return tablesFiles
      

def _generateCSVTo(targetFileName, dataSource = None, ignoreRows = 6):
    """
    targetFileName ::= where to write the CSV
    dataSource     ::= the scraped HTML file generated by pyavka.sh
    ignoreRows     ::= heuristic for skipping the first row headers in the
                       dataSource.
    """
    rowCount = 0
    rows     = list()
    soup     = BeautifulSoup(open(dataSource).read(), 'html.parser')
    table    = soup.find('table')

    print('processing %s...' % dataSource)

    for tableRow in table.find_all('tr'):
        rowCount += 1

        if rowCount > ignoreRows:
            row     = list()
            for column in tableRow.find_all('td'):
                row.append(column.text.replace(',', '')) # number format comma
            
            if not len(row[0]):
                continue    # skip to next record
            
            rows.append(row[0:7])

    with open(targetFileName, 'w') as outputFile:
        csv.writer(outputFile, delimiter = '\t').writerows(rows)

    print('generated %s' % targetFileName)


def processHTML2CSV(dataLake = SITE_DATA):
    # TODO:  This whole thing needs to be cleaned up later.
    for _, tableSpec in detectHTMLTablesRegions(dataLake).items():
        if tableSpec['fileCSV']:
            _generateCSVTo(tableSpec['fileCSV'], tableSpec['fileHTML'], tableSpec['ignoreRows'])


def _main():
    processHTML2CSV()


# --- main ---

if '__main__' == __name__:
    _main()


#!/usr/bin/env python3
# See: https://github.com/VirusTrack/COVIDvu/blob/master/LICENSE 
# vim: set fileencoding=utf-8:


from bs4 import BeautifulSoup

from covidvu.pipeline.vujson import SITE_DATA
from covidvu.pipeline.vuupdate import SCRAPED_US_DATA
from covidvu.pipeline.vuupdate import SCRAPED_WORLD_DATA

import copy
import csv
import os


# --- constants ---

TABLES_FILES     = {
                    'LOCATION': { 
                        'fileCSV' : SCRAPED_WORLD_DATA, 
                        'fileHTML': None,
                        'ignoreRows': 6,
                        'tableHTML' : None,
                    },
                    'UNITED STATES': { 
                        'fileCSV' : SCRAPED_US_DATA, 
                        'fileHTML': None,
                        'ignoreRows': 5,
                        'tableHTML' : None,
                    },
                    'AUSTRALIA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'MAINLAND CHINA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'CANADA': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                    'MUNDO HISPANO': { 
                        'fileCSV' : None, 
                        'fileHTML': None,
                        'tableHTML' : None,
                    },
                   }


# +++ functions +++

def detectHTMLTablesRegions(dataLake = SITE_DATA):
    """
    Returns a dictionary of file names like:

    {
        'Global': 'table-00.html',
        'US'    " 'table-01.html',
    }
    """
    prefix      = 'table-'
    filesList   = [ os.path.join(dataLake, fileName) for fileName in os.listdir(dataLake) if prefix in fileName and '.swp' not in fileName ]
    tablesFiles = copy.deepcopy(TABLES_FILES)

    for fileName in filesList:
        table = BeautifulSoup(open(fileName).read(), 'html.parser')

        for row in table.find_all('tr'):
            column = row.find('td')
            if not column:
                continue
            
            text = column.text.strip().upper()

            if text in tablesFiles:
                tablesFiles[text]['fileHTML'] = fileName
                break

    return tablesFiles
      

def _generateCSVTo(targetFileName, dataSource = None, ignoreRows = 6):
    """
    targetFileName ::= where to write the CSV
    dataSource     ::= the scraped HTML file generated by pyavka.sh
    ignoreRows     ::= heuristic for skipping the first row headers in the
                       dataSource.
    """
    rowCount = 0
    rows     = list()
    soup     = BeautifulSoup(open(dataSource).read(), 'html.parser')
    table    = soup.find('table')

    print('processing %s...' % dataSource)

    for tableRow in table.find_all('tr'):
        rowCount += 1

        if rowCount > ignoreRows:
            row     = list()
            for column in tableRow.find_all('td'):
                row.append(column.text.replace(',', '')) # number format comma
            
            if not len(row[0]):
                continue    # skip to next record
            
            rows.append(row[0:7])

    with open(targetFileName, 'w') as outputFile:
        csv.writer(outputFile, delimiter = '\t').writerows(rows)

    print('generated %s' % targetFileName)


def processHTML2CSV(dataLake = SITE_DATA):
    # TODO:  This whole thing needs to be cleaned up later.
    for _, tableSpec in detectHTMLTablesRegions(dataLake).items():
        if tableSpec['fileCSV']:
            _generateCSVTo(tableSpec['fileCSV'], tableSpec['fileHTML'], tableSpec['ignoreRows'])


def _main():
    processHTML2CSV()


# --- main ---

if '__main__' == __name__:
    _main()
#!/usr/bin/env python3 # See: https://github.com/VirusTrack/COVIDvu/blob/master/LICENSE # vim: set fileencoding=utf-8: # --- constants --- # +++ functions +++ Returns a dictionary of file names like:      {         'Global': 'table-00.html',         'US'    " 'table-01.html',     } targetFileName ::= where to write the CSV     dataSource     ::= the scraped HTML file generated by pyavka.sh     ignoreRows     ::= heuristic for skipping the first row headers in the                        dataSource. # number format comma # skip to next record # TODO:  This whole thing needs to be cleaned up later. # --- main ---