# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd

# # Get UUID data

csv_url_uids = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A78411ef4-bd3e-4cdd-b571-551d857bf768'
uids_df = pd.read_csv(csv_url_uids)

# # Get Years Data

csv_url_1964 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Adaa56be3-7d34-4c84-b0d0-75e25174f9a8'
csv_url_1967 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Af3465e2b-53be-46c6-ae45-2d56b5ab2f07'
csv_url_1968 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A32aa6111-06fa-4c3a-8cc9-aa21f2e4f575'
csv_url_1969 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A410392a8-02a6-4a28-8a27-bd245ebcb2f6'
csv_url_1970 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2f6b1165-26a4-4f89-a40a-22bc6473b754'
csv_url_1971 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aad152592-8fd0-4386-9787-cc4ae309a347'
csv_url_1972 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A52e6ef24-fabe-42f3-addd-d9da6409e653'
csv_url_1973 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A1d0a80d3-499e-4e71-af93-d92ee33f34dc'
csv_url_1974 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A1db2abd5-a11d-4411-9338-17979a0cab62'
csv_url_1975 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2b158a0a-b031-4fe2-9ede-7a3259b00b7d'
csv_url_1976 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aab62c166-e7f5-46a9-ae9f-d31910c3fb2d'
csv_url_1977 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A30084727-bdde-4c38-9e68-fa6b89a14cfb'
csv_url_1978 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Acd3a7adc-525a-402b-bdf4-dc78c0b341cd'
csv_url_1979 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A4c044daf-de6b-4a14-b75b-e2fe8256fc58'
csv_url_1980 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A573ae92d-6575-4acf-9224-c87a2078aec6'
csv_url_1981 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Ad9349c12-0678-4eb6-94f1-1b74f5dd8653'
csv_url_1984 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A51c04db8-a1d3-4f9a-8e89-cea4b481c21d'
csv_url_1986 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A23b13871-21f3-4082-b678-f3d6dc60784c'
csv_url_1987 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A711ec14b-5233-445d-9769-e0b384ae9058'
csv_url_1988 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2f3637d3-2749-481f-8af7-d00681a82941'
csv_url_1990 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A79a4c37b-262f-4741-ae57-93289a88547b'
csv_url_1991 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A3d4e3ecc-32cc-4d89-acdc-e87521f5c82a'
csv_url_1992 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aa15c869b-f557-4ec8-ae14-677ce17160bf'
csv_url_1993 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Af1b29736-92f3-4a74-acd7-b92d9b51c061'
csv_url_1994 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A34e03d3b-4951-4243-8e53-0e72053d45e8'
csv_url_1995 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Ac0de3c8f-952b-4ce9-89cc-ec8aaf6edbb2'
csv_url_1996 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A47e56615-1be0-4eff-a68e-cfad6ba583df'
csv_url_1997 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aa78ddb65-c20f-4c47-bacb-24d128f6a94c'

csv_url_years = [csv_url_1964, csv_url_1967, csv_url_1968, csv_url_1969,
                 csv_url_1970, csv_url_1971, csv_url_1972, csv_url_1973, 
                 csv_url_1974, csv_url_1975, csv_url_1976, csv_url_1977, 
                 csv_url_1978, csv_url_1979, csv_url_1980, csv_url_1981,
                 csv_url_1984, csv_url_1986, csv_url_1987, csv_url_1988,
                 csv_url_1990, csv_url_1991, csv_url_1992, csv_url_1993,
                 csv_url_1994, csv_url_1995, csv_url_1996, csv_url_1997]

years_df = pd.DataFrame()
for i in csv_url_years:
    data       = pd.read_csv(i)
    years_df   = years_df.append(data)

years_df

# # Check UUIDs data for inconsistencies

# ## 1. Remove images without 'NAGAP' text

src = uids_df.dropna()
src.head()

len(src[src.fileName_tn.str.contains('NAGAP')]) - len(src)

len(src[src.fileName_jpeg.str.contains('NAGAP')]) - len(src)

len(src[src.fileName_tiff.str.contains('NAGAP')]) - len(src)

# 156 thumbnail images do not contain string NAGAP 
#
# 166 tiff images do not contain string NAGAP
#
# 156 jpeg images do not contain string NAGAP
#
# Filter images without tiff file names containing NAGAP

src = src[src.fileName_tiff.str.contains('NAGAP')]
len(src[src.fileName_tn.str.contains('NAGAP')]), len(src[src.fileName_jpeg.str.contains('NAGAP')]), len(src)

# Now, all file names have NAGAP

# ## Look for Inconsistencies in Year

src.head()

# ### File name differences between tiffs and jpegs

(src[
    src.fileName_tiff.str.lower().apply(lambda x: x.replace('.tif', '')) != src.fileName_jpeg.str.lower().apply(lambda x: x.replace('.jpg', ''))
])

# ### File name differences between jpegs and thumbnail jpegs

(src[
    src.fileName_jpeg.str.lower().apply(lambda x: x.replace('.jpg', '')) != src.fileName_tn.str.lower().apply(lambda x: x.replace('_tn.jpg', ''))
])

# ### File name differences between tiffs and thumbnail jpegs

(src[
    src.fileName_tiff.str.lower().apply(lambda x: x.replace('.tif', '')) != src.fileName_tn.str.lower().apply(lambda x: x.replace('_tn.jpg', ''))
])

# FRAME NUMBER INDICATED BY DIFFERENT FILENAMES IS DIFFERENT??
#
# NEED TO FIGURE OUT WHICH IS THE ACTUAL FRAME NUMBER - THAT INDICATED BY WHICH FILENAME??

# ### Differences in Years identified by filename and Year columns

# +
# last two digits of Year column matching with fileName year
src['year_from_filename'] = svalue_counts_tiff.str[6:8] == 
src.Year.astype('str').str[2:] 

src['match'] = 
# -



src[~src.match]

uids_df['fileName'] = uids_df['fileName_jpeg'].str[:-4]

years_df['fileName'] = 'NAGAP_' + \
                       years_df['Roll'].astype(str) + '_' + \
                       years_df['Frame'].astype(str)

# +
y = list(years_df['fileName'].values)

filenames = []

for i in y:
    if len(i) == 12:
        a = i.split('_')
        i = a[0]+'_'+a[1]+'_00'+a[2]
        filenames.append(i)


    elif len(i) == 13:
        a = i.split('_')
        i = a[0]+'_'+ a[1]+'_0'+a[2]
        filenames.append(i)

    else:
        filenames.append(i)
# -

years_df['fileName'] = filenames

db = pd.merge(uids_df, years_df, on='fileName',indicator=True, how='outer')

db = db[['Year', 
         'Date', 
         'Location', 
         'Latitude',
         'Longitude',
         'Altitude',
         'fileName',
         'pid_tn',
         'pid_jpeg', 
         'pid_tiff', 
         '_merge',]]

db = db[db['fileName'].notnull()]

db = db[(db['fileName'].str.startswith('N')) | (db['fileName'].str.startswith('A'))]

# ## Look for inconsistencies

db['mismatch'] = (db.Year.fillna(0).astype('int')%100).astype('str')  == db['fileName'].apply(lambda x: x[6:8])

db[db.mismatch == False]

# ## Save to file

db.to_csv('glacier_names_uids.csv', index=False)

# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd

# # Get UUID data

csv_url_uids = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A78411ef4-bd3e-4cdd-b571-551d857bf768'
uids_df = pd.read_csv(csv_url_uids)

# # Get Years Data

csv_url_1964 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Adaa56be3-7d34-4c84-b0d0-75e25174f9a8'
csv_url_1967 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Af3465e2b-53be-46c6-ae45-2d56b5ab2f07'
csv_url_1968 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A32aa6111-06fa-4c3a-8cc9-aa21f2e4f575'
csv_url_1969 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A410392a8-02a6-4a28-8a27-bd245ebcb2f6'
csv_url_1970 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2f6b1165-26a4-4f89-a40a-22bc6473b754'
csv_url_1971 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aad152592-8fd0-4386-9787-cc4ae309a347'
csv_url_1972 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A52e6ef24-fabe-42f3-addd-d9da6409e653'
csv_url_1973 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A1d0a80d3-499e-4e71-af93-d92ee33f34dc'
csv_url_1974 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A1db2abd5-a11d-4411-9338-17979a0cab62'
csv_url_1975 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2b158a0a-b031-4fe2-9ede-7a3259b00b7d'
csv_url_1976 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aab62c166-e7f5-46a9-ae9f-d31910c3fb2d'
csv_url_1977 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A30084727-bdde-4c38-9e68-fa6b89a14cfb'
csv_url_1978 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Acd3a7adc-525a-402b-bdf4-dc78c0b341cd'
csv_url_1979 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A4c044daf-de6b-4a14-b75b-e2fe8256fc58'
csv_url_1980 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A573ae92d-6575-4acf-9224-c87a2078aec6'
csv_url_1981 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Ad9349c12-0678-4eb6-94f1-1b74f5dd8653'
csv_url_1984 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A51c04db8-a1d3-4f9a-8e89-cea4b481c21d'
csv_url_1986 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A23b13871-21f3-4082-b678-f3d6dc60784c'
csv_url_1987 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A711ec14b-5233-445d-9769-e0b384ae9058'
csv_url_1988 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A2f3637d3-2749-481f-8af7-d00681a82941'
csv_url_1990 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A79a4c37b-262f-4741-ae57-93289a88547b'
csv_url_1991 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A3d4e3ecc-32cc-4d89-acdc-e87521f5c82a'
csv_url_1992 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aa15c869b-f557-4ec8-ae14-677ce17160bf'
csv_url_1993 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Af1b29736-92f3-4a74-acd7-b92d9b51c061'
csv_url_1994 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A34e03d3b-4951-4243-8e53-0e72053d45e8'
csv_url_1995 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Ac0de3c8f-952b-4ce9-89cc-ec8aaf6edbb2'
csv_url_1996 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3A47e56615-1be0-4eff-a68e-cfad6ba583df'
csv_url_1997 = 'https://arcticdata.io/metacat/d1/mn/v2/object/urn%3Auuid%3Aa78ddb65-c20f-4c47-bacb-24d128f6a94c'

csv_url_years = [csv_url_1964, csv_url_1967, csv_url_1968, csv_url_1969,
                 csv_url_1970, csv_url_1971, csv_url_1972, csv_url_1973, 
                 csv_url_1974, csv_url_1975, csv_url_1976, csv_url_1977, 
                 csv_url_1978, csv_url_1979, csv_url_1980, csv_url_1981,
                 csv_url_1984, csv_url_1986, csv_url_1987, csv_url_1988,
                 csv_url_1990, csv_url_1991, csv_url_1992, csv_url_1993,
                 csv_url_1994, csv_url_1995, csv_url_1996, csv_url_1997]

years_df = pd.DataFrame()
for i in csv_url_years:
    data       = pd.read_csv(i)
    years_df   = years_df.append(data)

years_df

# # Check UUIDs data for inconsistencies

# ## 1. Remove images without 'NAGAP' text

src = uids_df.dropna()
src.head()

len(src[src.fileName_tn.str.contains('NAGAP')]) - len(src)

len(src[src.fileName_jpeg.str.contains('NAGAP')]) - len(src)

len(src[src.fileName_tiff.str.contains('NAGAP')]) - len(src)

# 156 thumbnail images do not contain string NAGAP 
#
# 166 tiff images do not contain string NAGAP
#
# 156 jpeg images do not contain string NAGAP
#
# Filter images without tiff file names containing NAGAP

src = src[src.fileName_tiff.str.contains('NAGAP')]
len(src[src.fileName_tn.str.contains('NAGAP')]), len(src[src.fileName_jpeg.str.contains('NAGAP')]), len(src)

# Now, all file names have NAGAP

# ## Look for Inconsistencies in Year

src.head()

# ### File name differences between tiffs and jpegs

(src[
    src.fileName_tiff.str.lower().apply(lambda x: x.replace('.tif', '')) != src.fileName_jpeg.str.lower().apply(lambda x: x.replace('.jpg', ''))
])

# ### File name differences between jpegs and thumbnail jpegs

(src[
    src.fileName_jpeg.str.lower().apply(lambda x: x.replace('.jpg', '')) != src.fileName_tn.str.lower().apply(lambda x: x.replace('_tn.jpg', ''))
])

# ### File name differences between tiffs and thumbnail jpegs

(src[
    src.fileName_tiff.str.lower().apply(lambda x: x.replace('.tif', '')) != src.fileName_tn.str.lower().apply(lambda x: x.replace('_tn.jpg', ''))
])

# FRAME NUMBER INDICATED BY DIFFERENT FILENAMES IS DIFFERENT??
#
# NEED TO FIGURE OUT WHICH IS THE ACTUAL FRAME NUMBER - THAT INDICATED BY WHICH FILENAME??

# ### Differences in Years identified by filename and Year columns

# +
# last two digits of Year column matching with fileName year
src['year_from_filename'] = svalue_counts_tiff.str[6:8] == 
src.Year.astype('str').str[2:] 

src['match'] = 
# -



src[~src.match]

uids_df['fileName'] = uids_df['fileName_jpeg'].str[:-4]

years_df['fileName'] = 'NAGAP_' + \
                       years_df['Roll'].astype(str) + '_' + \
                       years_df['Frame'].astype(str)

# +
y = list(years_df['fileName'].values)

filenames = []

for i in y:
    if len(i) == 12:
        a = i.split('_')
        i = a[0]+'_'+a[1]+'_00'+a[2]
        filenames.append(i)


    elif len(i) == 13:
        a = i.split('_')
        i = a[0]+'_'+ a[1]+'_0'+a[2]
        filenames.append(i)

    else:
        filenames.append(i)
# -

years_df['fileName'] = filenames

db = pd.merge(uids_df, years_df, on='fileName',indicator=True, how='outer')

db = db[['Year', 
         'Date', 
         'Location', 
         'Latitude',
         'Longitude',
         'Altitude',
         'fileName',
         'pid_tn',
         'pid_jpeg', 
         'pid_tiff', 
         '_merge',]]

db = db[db['fileName'].notnull()]

db = db[(db['fileName'].str.startswith('N')) | (db['fileName'].str.startswith('A'))]

# ## Look for inconsistencies

db['mismatch'] = (db.Year.fillna(0).astype('int')%100).astype('str')  == db['fileName'].apply(lambda x: x[6:8])

db[db.mismatch == False]

# ## Save to file

db.to_csv('glacier_names_uids.csv', index=False)

# --- # jupyter: #   jupytext: #     text_representation: #       extension: .py #       format_name: light #       format_version: '1.5' #       jupytext_version: 1.5.2 #   kernelspec: #     display_name: Python 3 #     language: python #     name: python3 # --- # # Get UUID data # # Get Years Data # # Check UUIDs data for inconsistencies # ## 1. Remove images without 'NAGAP' text # 156 thumbnail images do not contain string NAGAP # # 166 tiff images do not contain string NAGAP # # 156 jpeg images do not contain string NAGAP # # Filter images without tiff file names containing NAGAP # Now, all file names have NAGAP # ## Look for Inconsistencies in Year # ### File name differences between tiffs and jpegs # ### File name differences between jpegs and thumbnail jpegs # ### File name differences between tiffs and thumbnail jpegs # FRAME NUMBER INDICATED BY DIFFERENT FILENAMES IS DIFFERENT?? # # NEED TO FIGURE OUT WHICH IS THE ACTUAL FRAME NUMBER - THAT INDICATED BY WHICH FILENAME?? # ### Differences in Years identified by filename and Year columns # + # last two digits of Year column matching with fileName year # - # + # - # ## Look for inconsistencies # ## Save to file