<reponame>bvid/CameraTraps
#
# get_lila_category_list.py
#
# Example of making a text file listing all category names in specific LILA datasets
#

#%% Constants and imports

import json
import urllib.request
import tempfile
import zipfile
import os

from urllib.parse import urlparse

# LILA camera trap master metadata file
metadata_url = 'http://lila.science/wp-content/uploads/2020/03/lila_sas_urls.txt'

# array to fill for output
category_list = []

## datasets and categories to look at

# if False, will only collect data for categories in datasets_of_interest
use_all_datasets = True 

# only need if restrict_category is false
datasets_of_interest = [] 

# We'll write images, metadata downloads, and temporary files here
lila_local_base = r'g:\temp\lila'

output_dir = os.path.join(lila_local_base,'lila_categories_list')
os.makedirs(output_dir,exist_ok=True)

metadata_dir = os.path.join(lila_local_base,'metadata')
os.makedirs(metadata_dir,exist_ok=True)

output_file = os.path.join(output_dir,'category_list.txt')


#%% Support functions

def download_url(url, destination_filename=None, force_download=False, verbose=True):
    """
    Download a URL (defaulting to a temporary file)
    """
    
    if destination_filename is None:
        temp_dir = os.path.join(tempfile.gettempdir(),'lila')
        os.makedirs(temp_dir,exist_ok=True)
        url_as_filename = url.replace('://', '_').replace('.', '_').replace('/', '_')
        destination_filename =             os.path.join(temp_dir,url_as_filename)
            
    if (not force_download) and (os.path.isfile(destination_filename)):
        print('Bypassing download of already-downloaded file {}'.format(os.path.basename(url)))
        return destination_filename
    
    if verbose:
        print('Downloading file {} to {}'.format(os.path.basename(url),destination_filename),end='')
    
    os.makedirs(os.path.dirname(destination_filename),exist_ok=True)
    urllib.request.urlretrieve(url, destination_filename)  
    assert(os.path.isfile(destination_filename))
    
    if verbose:
        nBytes = os.path.getsize(destination_filename)    
        print('...done, {} bytes.'.format(nBytes))
        
    return destination_filename


def download_relative_filename(url, output_base, verbose=False):
    """
    Download a URL to output_base, preserving relative path
    """
    
    p = urlparse(url)
    # remove the leading '/'
    assert p.path.startswith('/'); relative_filename = p.path[1:]
    destination_filename = os.path.join(output_base,relative_filename)
    download_url(url, destination_filename, verbose=verbose)
    

def unzip_file(input_file, output_folder=None):
    """
    Unzip a zipfile to the specified output folder, defaulting to the same location as
    the input file    
    """
    
    if output_folder is None:
        output_folder = os.path.dirname(input_file)
        
    with zipfile.ZipFile(input_file, 'r') as zf:
        zf.extractall(output_folder)


#%% Download and parse the metadata file

# Put the master metadata file in the same folder where we're putting images
p = urlparse(metadata_url)
metadata_filename = os.path.join(metadata_dir,os.path.basename(p.path))

# Read lines from the master metadata file
with open(metadata_filename,'r') as f:
    metadata_lines = f.readlines()
metadata_lines = [s.strip() for s in metadata_lines]

# Parse those lines into a table
metadata_table = {}

for s in metadata_lines:
    
    if len(s) == 0 or s[0] == '#':
        continue
    
    # Each line in this file is name/sas_url/json_url/[bbox_json_url]
    tokens = s.split(',')
    assert len(tokens) == 4
    ds_name = tokens[0].strip()
    url_mapping = {'sas_url':tokens[1],'json_url':tokens[2]}
    metadata_table[ds_name] = url_mapping
    
    # Create a separate entry for bounding boxes if they exist
    if len(tokens[3].strip()) > 0:
        print('Adding bounding box dataset for {}'.format(ds_name))
        bbox_url_mapping = {'sas_url':tokens[1],'json_url':tokens[3]}
        metadata_table[tokens[0]+'_bbox'] = bbox_url_mapping
        assert 'https' in bbox_url_mapping['json_url']

    assert 'https' not in tokens[0]
    assert 'https' in url_mapping['sas_url']
    assert 'https' in url_mapping['json_url']


#%% Download and extract metadata for the datasets we're interested in

if use_all_datasets: datasets_of_interest = list(metadata_table.keys())
for ds_name in datasets_of_interest:
    
    assert ds_name in metadata_table
    json_url = metadata_table[ds_name]['json_url']
    
    p = urlparse(json_url)
    json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
    download_url(json_url, json_filename)
    
    # Unzip if necessary
    if json_filename.endswith('.zip'):
        
        with zipfile.ZipFile(json_filename,'r') as z:
            files = z.namelist()
        assert len(files) == 1
        unzipped_json_filename = os.path.join(metadata_dir,files[0])
        if not os.path.isfile(unzipped_json_filename):
            unzip_file(json_filename,metadata_dir)        
        else:
            print('{} already unzipped'.format(unzipped_json_filename))
        json_filename = unzipped_json_filename
    
    metadata_table[ds_name]['json_filename'] = json_filename
    
# ...for each dataset of interest


#%% Get category names

for ds_name in datasets_of_interest:
    
    print('Finding categories in {}'.format(ds_name))
    
    json_filename = metadata_table[ds_name]['json_filename']
    sas_url = metadata_table[ds_name]['sas_url']
    
    base_url = sas_url.split('?')[0]    
    assert not base_url.endswith('/')
    
    # Open the metadata file    
    with open(json_filename, 'r') as f:
        data = json.load(f)
    
    # Collect list of categories and mappings to category name
    categories = data['categories']
    category_ids = [c['id'] for c in categories]
    for c in categories:
        c['name'] = c['name'].lower()
        category_id_to_name = {c['id']:c['name'] for c in categories}
    
    # Append category to categories_list
    for category_id in category_ids:
        category_name = category_id_to_name[category_id]
        if category_name not in category_list: category_list.append(category_name)


#%% Save category names to file

with open(output_file, 'w') as txt_file:
    for line in category_list:
        txt_file.write(line + '\n')

#
# get_lila_category_list.py
#
# Example of making a text file listing all category names in specific LILA datasets
#

#%% Constants and imports

import json
import urllib.request
import tempfile
import zipfile
import os

from urllib.parse import urlparse

# LILA camera trap master metadata file
metadata_url = 'http://lila.science/wp-content/uploads/2020/03/lila_sas_urls.txt'

# array to fill for output
category_list = []

## datasets and categories to look at

# if False, will only collect data for categories in datasets_of_interest
use_all_datasets = True 

# only need if restrict_category is false
datasets_of_interest = [] 

# We'll write images, metadata downloads, and temporary files here
lila_local_base = r'g:\temp\lila'

output_dir = os.path.join(lila_local_base,'lila_categories_list')
os.makedirs(output_dir,exist_ok=True)

metadata_dir = os.path.join(lila_local_base,'metadata')
os.makedirs(metadata_dir,exist_ok=True)

output_file = os.path.join(output_dir,'category_list.txt')


#%% Support functions

def download_url(url, destination_filename=None, force_download=False, verbose=True):
    """
    Download a URL (defaulting to a temporary file)
    """
    
    if destination_filename is None:
        temp_dir = os.path.join(tempfile.gettempdir(),'lila')
        os.makedirs(temp_dir,exist_ok=True)
        url_as_filename = url.replace('://', '_').replace('.', '_').replace('/', '_')
        destination_filename =             os.path.join(temp_dir,url_as_filename)
            
    if (not force_download) and (os.path.isfile(destination_filename)):
        print('Bypassing download of already-downloaded file {}'.format(os.path.basename(url)))
        return destination_filename
    
    if verbose:
        print('Downloading file {} to {}'.format(os.path.basename(url),destination_filename),end='')
    
    os.makedirs(os.path.dirname(destination_filename),exist_ok=True)
    urllib.request.urlretrieve(url, destination_filename)  
    assert(os.path.isfile(destination_filename))
    
    if verbose:
        nBytes = os.path.getsize(destination_filename)    
        print('...done, {} bytes.'.format(nBytes))
        
    return destination_filename


def download_relative_filename(url, output_base, verbose=False):
    """
    Download a URL to output_base, preserving relative path
    """
    
    p = urlparse(url)
    # remove the leading '/'
    assert p.path.startswith('/'); relative_filename = p.path[1:]
    destination_filename = os.path.join(output_base,relative_filename)
    download_url(url, destination_filename, verbose=verbose)
    

def unzip_file(input_file, output_folder=None):
    """
    Unzip a zipfile to the specified output folder, defaulting to the same location as
    the input file    
    """
    
    if output_folder is None:
        output_folder = os.path.dirname(input_file)
        
    with zipfile.ZipFile(input_file, 'r') as zf:
        zf.extractall(output_folder)


#%% Download and parse the metadata file

# Put the master metadata file in the same folder where we're putting images
p = urlparse(metadata_url)
metadata_filename = os.path.join(metadata_dir,os.path.basename(p.path))

# Read lines from the master metadata file
with open(metadata_filename,'r') as f:
    metadata_lines = f.readlines()
metadata_lines = [s.strip() for s in metadata_lines]

# Parse those lines into a table
metadata_table = {}

for s in metadata_lines:
    
    if len(s) == 0 or s[0] == '#':
        continue
    
    # Each line in this file is name/sas_url/json_url/[bbox_json_url]
    tokens = s.split(',')
    assert len(tokens) == 4
    ds_name = tokens[0].strip()
    url_mapping = {'sas_url':tokens[1],'json_url':tokens[2]}
    metadata_table[ds_name] = url_mapping
    
    # Create a separate entry for bounding boxes if they exist
    if len(tokens[3].strip()) > 0:
        print('Adding bounding box dataset for {}'.format(ds_name))
        bbox_url_mapping = {'sas_url':tokens[1],'json_url':tokens[3]}
        metadata_table[tokens[0]+'_bbox'] = bbox_url_mapping
        assert 'https' in bbox_url_mapping['json_url']

    assert 'https' not in tokens[0]
    assert 'https' in url_mapping['sas_url']
    assert 'https' in url_mapping['json_url']


#%% Download and extract metadata for the datasets we're interested in

if use_all_datasets: datasets_of_interest = list(metadata_table.keys())
for ds_name in datasets_of_interest:
    
    assert ds_name in metadata_table
    json_url = metadata_table[ds_name]['json_url']
    
    p = urlparse(json_url)
    json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
    download_url(json_url, json_filename)
    
    # Unzip if necessary
    if json_filename.endswith('.zip'):
        
        with zipfile.ZipFile(json_filename,'r') as z:
            files = z.namelist()
        assert len(files) == 1
        unzipped_json_filename = os.path.join(metadata_dir,files[0])
        if not os.path.isfile(unzipped_json_filename):
            unzip_file(json_filename,metadata_dir)        
        else:
            print('{} already unzipped'.format(unzipped_json_filename))
        json_filename = unzipped_json_filename
    
    metadata_table[ds_name]['json_filename'] = json_filename
    
# ...for each dataset of interest


#%% Get category names

for ds_name in datasets_of_interest:
    
    print('Finding categories in {}'.format(ds_name))
    
    json_filename = metadata_table[ds_name]['json_filename']
    sas_url = metadata_table[ds_name]['sas_url']
    
    base_url = sas_url.split('?')[0]    
    assert not base_url.endswith('/')
    
    # Open the metadata file    
    with open(json_filename, 'r') as f:
        data = json.load(f)
    
    # Collect list of categories and mappings to category name
    categories = data['categories']
    category_ids = [c['id'] for c in categories]
    for c in categories:
        c['name'] = c['name'].lower()
        category_id_to_name = {c['id']:c['name'] for c in categories}
    
    # Append category to categories_list
    for category_id in category_ids:
        category_name = category_id_to_name[category_id]
        if category_name not in category_list: category_list.append(category_name)


#%% Save category names to file

with open(output_file, 'w') as txt_file:
    for line in category_list:
        txt_file.write(line + '\n')
# # get_lila_category_list.py # # Example of making a text file listing all category names in specific LILA datasets # #%% Constants and imports # LILA camera trap master metadata file # array to fill for output ## datasets and categories to look at # if False, will only collect data for categories in datasets_of_interest # only need if restrict_category is false # We'll write images, metadata downloads, and temporary files here #%% Support functions Download a URL (defaulting to a temporary file) Download a URL to output_base, preserving relative path # remove the leading '/' Unzip a zipfile to the specified output folder, defaulting to the same location as     the input file #%% Download and parse the metadata file # Put the master metadata file in the same folder where we're putting images # Read lines from the master metadata file # Parse those lines into a table # Each line in this file is name/sas_url/json_url/[bbox_json_url] # Create a separate entry for bounding boxes if they exist #%% Download and extract metadata for the datasets we're interested in # Unzip if necessary # ...for each dataset of interest #%% Get category names # Open the metadata file # Collect list of categories and mappings to category name # Append category to categories_list #%% Save category names to file