<filename>sourceCodeRelease/1_individual/helpers.py
""" 
A bunch of helpers to assist with investigation 
Contains: 
    - mapping from freeform countries to country codes (ship from field)
    - extracting quantities and weight from a given item name
<NAME>, 2016
This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.

"""

import re
import sys

EU_free2fixed = {"A boring central-EU country" : "Other",
                 "A boring middle EU country" : "Other",
                 "A boring middle-EU country" : "Other",
                 "Amsterdam" : "NL", 
                 "Andorra" : "Other", 
                 "Austria" : "AT", 
                 "BELGIUM" : "BE",
                 "Belgium" : "BE",
                 "Belguim" : "BE",
                 "Bulgaria" : "BG", 
                 "Central Europe" : "Other",
                 "Centre Europe" : "Other",
                 "Croatia" : "HR", 
                 "Cyprus" : "CY", 
                 "Czech" : "CZ", 
                 "Czech Rep." : "CZ", 
                 "Czech Repubic" : "CZ", 
                 "Czech Republic" : "CZ", 
                 "Czech Republic (EU)" : "CZ", 
                 "Czech republic" : "CZ", 
                 "DENMARK" : "DK", 
                 "DK" : "DK", 
                 "Denmark" : "DK", 
                 "Denmark / UK (Top #4 Seller)" : "Other", 
                 "Deutschland" : "DE", 
                 "Deutschland/Germany" : "DE", 
                 "E.U" : "Other",
                 "E.U." : "Other",
                 "EU" : "Other",
                 "EU - FR" : "FR", 
                 "EU - NOT NETHERLAND !!" : "Other",
                 "EU - NOT NETHERLANDS !!" : "Other",
                 "EU / FINLAND / FRANCE" : "Other",
                 "EU / RETAILER AGENT" : "Other",
                 "EU in sealed envelope" : "Other",
                 "EU, USA, and Canada" : "Other",
                 "EUR" : "Other",
                 "EUROPE" : "Other",
                 "East-EU" : "Other",
                 "Eastern Europe" : "Other",
                 "England" : "UK",
                 "Estonia" : "EE", 
                 "Eu" : "Other",
                 "Europe" : "Other",
                 "Europe Union" : "Other",
                 "Europe or Depots" : "Other",
                 "Europe, not NL" : "Other",
                 "European" : "Other",
                 "European Country" : "Other",
                 "European Union" : "Other",
                 "European Union (NOT from Netherland)" : "Other",
                 "European Union (NOT from Netherlands)" : "Other",
                 "European union" : "Other",
                 "FIN" : "FI", 
                 "FR" : "FR",
                 "FRANCE" : "FR",
                 "FRANCE FULLZ WITH DOB!" : "FR",
                 "Finland" : "FI", 
                 "Finland, Suomi" : "FI", 
                 "France" : "FR", 
                 "France - EU" : "FR", 
                 "France / French" : "FR", 
                 "France/Europe" : "FR", 
                 "French Guiana" : "FR", 
                 "French Southern Territories" : "FR", 
                 "GER" : "DE", 
                 "GERMANY" : "DE", 
                 "Ger" : "DE", 
                 "German" : "DE", 
                 "Germany" : "DE", 
                 "Germany (Monday and Friday)" : "DE", 
                 "Germany (Monday and Thursday)" : "DE", 
                 "Germany Monday and Thursday" : "DE", 
                 "Germany and Belgium" : "Other", 
                 "Greece" : "GR", 
                 "HOLLAND" : "NL", 
                 "Holland" : "NL", 
                 "Hollandia" : "NL", 
                 "Hungary" : "HU",
                 "ITALY" : "IT", 
                 "Ireland" : "IE", 
                 "Italy" : "IT", 
                 "Latvia" : "LV", 
                 "Lithuania" : "LT", 
                 "Luxembourg" : "LU", 
                 "Malta" : "MT",  
                 "Monaco" : "Other", 
                 "NL" : "NL", 
                 "NL / Belgium" : "Other", 
                 "NL / DE" : "Other", 
                 "Netehrlands" : "NL", 
                 "Netherland" : "NL", 
                 "Netherlands" : "NL", 
                 "Netherlands & Germany" : "Other",
                 "Netherlands / Belgium" : "Other",
                 "Netherlands / Germany" : "Other",
                 "Netherlands and Spain" : "Other",
                 "Netherlands or Germany" : "Other",
                 "Nl" : "NL", 
                 "Norway" : "NO", 
                 "POLAND" : "PL", 
                 "Poland" : "PL", 
                 "Portugal" : "PT", 
                 "Reunion" : "FR", 
                 "Reunion (FR)" : "FR", 
                 "Romania" : "RO", 
                 "San Marino" : "Other", 
                 "SWE" : "SE", 
                 "SWEDEN" : "SE", 
                 "Saint Martin (FR)" : "FR", 
                 "Scandinavia" : "Other", 
                 "Slovakia" : "SK", 
                 "Slovenia" : "SI",
                 "Slovenia ( European union country )" : "SI",
                 "Slovenia - EU" : "SI",
                 "Spain" : "ES",
                 "Spain or UK" : "Other", 
                 "Sweden" : "SE", 
                 "Sweden/Europe" : "SE", 
                 "THE NETHERLANDS" : "NL", 
                 "The Netherlands" : "NL", 
                 "The Nettherlands" : "NL", 
                 "Turkey" : "TR", 
                 "U.K" : "UK",
                 "UK" : "UK",
                 "UK and IRELAND" : "Other",
                 "UK and Ireland" : "Other",
                 "UK or Spain" : "Other",
                 "UNITED KINGDOM" : "UK",
                 "Uk"  : "UK",
                 "United KIngdom" : "UK",
                 "United Kingdom" : "UK",
                 "United kingdom" : "UK",
                 "Untied Kingdom" : "UK",
                 "Western europe" : "Other", 
                 "eu" : "Other", 
                 "europe" : "Other", 
                 "european union" : "Other", 
                 "fin" : "FI",
                 "finland" : "FI",
                 "france" : "FR", 
                 "germany" : "DE", 
                 "holland" : "NL", 
                 "netherlands" : "NL", 
                 "nl" : "NL", 
                 "nlgermany" : "Other", 
                 "the Netherlands" : "NL", 
                 "the netherlands" : "NL", 
                 "uk" : "UK", 
                 "united kingdom" : "UK",
                 "Europe except Scandinavia": "Other", 
                 "Belgium/Netherlands": "Other", 
                 "Berlin, Germany": "DE",
                 "Central Europe": "Other",
                 "Czechia, Slovakia": "Other",
                 "DE,AT,CH,DK": "Other", 
                 "DEUTSCHLAND, GERMANY": "DE",
                 "DEUTSCHLAND, GERMANY, EUROPE": "DE",
                 "Danmark": "DK",
                 "Danmark, Spain": "Other", 
                 "E.U.": "Other", 
                 "ESPAÃA": "ES", 
                 "EU": "Other", 
                 "EU - Poland": "PL",
                 "EUROPE": "Other",
                 "EspaÃ±a": "ES",
                 "Europa": "Other",
                 "Europe": "Other",
                 "Germany + Netherlands": "Other",
                 "Germany + netherlands": "Other",
                 "Germany Netherlands": "Other",
                 "Germany, Austria": "Other",
                 "Germany, Czech Rep., Poland": "Other",
                 "Germany, Netherlands or United Kingdom": "Other",
                 "Germany, Netherlands or United Kingdom ( Your's choice )": "Other",
                 "Germany/Deutschland": "Other",
                 "Holland Belgium Germany": "Other",
                 "Holland, Belgium": "Other",
                 "Milan": "IT", 
                 "NETHERLANDS": "NL",
                 "Netherlands, holland": "NL",
                 "Netherlands/Belgium": "Other", 
                 "Netherlands/Germany": "Other",
                 "SPAIN": "ES", 
                 "SWISS to SWISS only!": "CH",
                 "Slovakia Czechia": "Other", 
                 "Slovakia, Czechia": "Other", 
                 "Swiss to Swiss ONLY": "CH",
                 "Swiss to Swiss ONLY!!!": "CH",
                 "TURKEY": "TR", 
                 "The Netherlands": "NL",
                 "The Netherlands / Germany": "Other",
                 "The netherlands": "NL", 
                 "U.K/PL": "Other",
                 "UK to WW": "UK", 
                 "UK/PL": "Other",
                 "Western Europe": "Other", 
                 "czech republic": "CZ", 
                 "e.u.": "Other", 
                 "eu": "Other", 
                 "europe": "Other", 
                 "france": "FR",
                 "germany": "DE",
                 "netherland": "NL",
                 "netherlands": "NL",
                 "switzerland": "CH",
                 "the Netherlands": "NL",
                 "u.k": "UK",
                 "uk": "UK", 
                 "uk vendor (❣◡❣)": "UK",
                 "united kingdom": "UK", 
                 "united kingdom to worldwide": "UK"
                 }

# because python time handling is... gross wrt timezones, 
# this is precomputed

date2unix = {"2014-12-26": 1419552000,
             "2014-12-27": 1419638400,
             "2014-12-28": 1419724800,
             "2014-12-29": 1419811200,
             "2014-12-30": 1419897600,
             "2014-12-31": 1419984000,
             "2015-01-01": 1420070400,
             "2015-01-02": 1420156800,
             "2015-01-03": 1420243200,
             "2015-01-04": 1420329600,
             "2015-01-05": 1420416000,
             "2015-01-06": 1420502400,
             "2015-01-07": 1420588800,
             "2015-01-08": 1420675200,
             "2015-01-09": 1420761600,
             "2015-01-10": 1420848000,
             "2015-01-11": 1420934400,
             "2015-01-12": 1421020800,
             "2015-01-13": 1421107200,
             "2015-01-14": 1421193600,
             "2015-01-15": 1421280000,
             "2015-01-16": 1421366400,
             "2015-01-17": 1421452800,
             "2015-01-18": 1421539200,
             "2015-01-19": 1421625600,
             "2015-01-20": 1421712000,
             "2015-01-21": 1421798400,
             "2015-01-22": 1421884800,
             "2015-01-23": 1421971200,
             "2015-01-24": 1422057600,
             "2015-01-25": 1422144000,
             "2015-01-26": 1422230400,
             "2015-01-27": 1422316800,
             "2015-01-28": 1422403200,
             "2015-01-29": 1422489600,
             "2015-01-30": 1422576000,
             "2015-01-31": 1422662400,
             "2015-02-01": 1422748800,
             "2015-02-02": 1422835200,
             "2015-02-03": 1422921600,
             "2015-02-04": 1423008000,
             "2015-02-05": 1423094400,
             "2015-02-06": 1423180800,
             "2015-02-07": 1423267200,
             "2015-02-08": 1423353600,
             "2015-02-09": 1423440000,
             "2015-02-10": 1423526400,
             "2015-02-11": 1423612800,
             "2015-02-12": 1423699200,
             "2015-02-13": 1423785600,
             "2015-02-14": 1423872000,
             "2015-02-15": 1423958400,
             "2015-02-16": 1424044800,
             "2015-02-17": 1424131200,
             "2015-02-18": 1424217600,
             "2015-02-19": 1424304000,
             "2015-02-20": 1424390400,
             "2015-02-21": 1424476800,
             "2015-02-22": 1424563200,
             "2015-02-23": 1424649600,
             "2015-02-24": 1424736000,
             "2015-02-25": 1424822400,
             "2015-02-26": 1424908800,
             "2015-02-27": 1424995200,
             "2015-02-28": 1425081600,
             "2015-03-01": 1425168000,
             "2015-03-02": 1425254400,
             "2015-03-03": 1425340800,
             "2015-03-04": 1425427200,
             "2015-03-05": 1425513600,
             "2015-03-06": 1425600000,
             "2015-03-07": 1425686400,
             "2015-03-08": 1425772800,
             "2015-03-09": 1425859200,
             "2015-03-10": 1425945600,
             "2015-03-11": 1426032000,
             "2015-03-12": 1426118400,
             "2015-03-13": 1426204800,
             "2015-03-14": 1426291200,
             "2015-03-15": 1426377600,
             "2015-03-16": 1426464000,
             "2015-03-17": 1426550400,
             "2015-03-18": 1426636800,
             "2015-03-19": 1426723200,
             "2015-03-20": 1426809600,
             "2015-03-21": 1426896000,
             "2015-03-22": 1426982400,
             "2015-03-23": 1427068800,
             "2015-03-24": 1427155200,
             "2015-03-25": 1427241600,
             "2015-03-26": 1427328000,
             "2015-03-27": 1427414400,
             "2015-03-28": 1427500800,
             "2015-03-29": 1427587200,
             "2015-03-30": 1427673600,
             "2015-03-31": 1427760000,
             "2015-04-01": 1427846400,
             "2015-04-02": 1427932800,
             "2015-04-03": 1428019200,
             "2015-04-04": 1428105600,
             "2015-04-05": 1428192000,
             "2015-04-06": 1428278400,
             "2015-04-07": 1428364800,
             "2015-04-08": 1428451200,
             "2015-04-09": 1428537600,
             "2015-04-10": 1428624000,
             "2015-04-11": 1428710400,
             "2015-04-12": 1428796800,
             "2015-04-13": 1428883200,
             "2015-04-14": 1428969600,
             "2015-04-15": 1429056000,
             "2015-04-16": 1429142400,
             "2015-04-17": 1429228800,
             "2015-04-18": 1429315200,
             "2015-04-19": 1429401600,
             "2015-04-20": 1429488000,
             "2015-04-21": 1429574400,
             "2015-04-22": 1429660800,
             "2015-04-23": 1429747200,
             "2015-04-24": 1429833600,
             "2015-04-25": 1429920000,
             "2015-04-26": 1430006400,
             "2015-04-27": 1430092800,
             "2015-04-28": 1430179200,
             "2015-04-29": 1430265600,
             "2015-04-30": 1430352000,
             "2015-05-01": 1430438400,
             "2015-05-02": 1430524800,
             "2015-05-03": 1430611200,
             "2015-05-04": 1430697600,
             "2015-05-05": 1430784000,
             "2015-05-06": 1430870400,
             "2015-05-07": 1430956800,
             "2015-05-08": 1431043200,
             "2015-05-09": 1431129600,
             "2015-05-10": 1431216000,
             "2015-05-11": 1431302400,
             "2015-05-12": 1431388800,
             "2015-05-13": 1431475200,
             "2015-05-14": 1431561600,
             "2015-05-15": 1431648000,
             "2015-05-16": 1431734400,
             "2015-05-17": 1431820800,
             "2015-05-18": 1431907200,
             "2015-05-19": 1431993600,
             "2015-05-20": 1432080000,
             "2015-05-21": 1432166400,
             "2015-05-22": 1432252800,
             "2015-05-23": 1432339200,
             "2015-05-24": 1432425600,
             "2015-05-25": 1432512000,
             "2015-05-26": 1432598400,
             "2015-05-27": 1432684800,
             "2015-05-28": 1432771200,
             "2015-05-29": 1432857600,
             "2015-05-30": 1432944000,
             "2015-05-31": 1433030400,
             "2015-06-01": 1433116800,
             "2015-06-02": 1433203200,
             "2015-06-03": 1433289600,
             "2015-06-04": 1433376000,
             "2015-06-05": 1433462400,
             "2015-06-06": 1433548800,
             "2015-06-07": 1433635200,
             "2015-06-08": 1433721600,
             "2015-06-09": 1433808000,
             "2015-06-10": 1433894400,
             "2015-06-11": 1433980800,
             "2015-06-12": 1434067200,
             "2015-06-13": 1434153600,
             "2015-06-14": 1434240000,
             "2015-06-15": 1434326400,
             "2015-06-16": 1434412800,
             "2015-06-17": 1434499200,
             "2015-06-18": 1434585600,
             "2015-06-19": 1434672000,
             "2015-06-20": 1434758400,
             "2015-06-21": 1434844800,
             "2015-06-22": 1434931200,
             "2015-06-23": 1435017600,
             "2015-06-24": 1435104000,
             "2015-06-25": 1435190400,
             "2015-06-26": 1435276800,
             "2015-06-27": 1435363200,
             "2015-06-28": 1435449600,
             "2015-06-29": 1435536000,
             "2015-06-30": 1435622400,
             "2015-07-01": 1435708800,
             "2015-07-02": 1435795200,
             "2015-07-03": 1435881600,
             "2015-07-04": 1435968000,
             "2015-07-05": 1436054400,
             "2015-07-06": 1436140800,
             "2015-07-07": 1436227200,
             "2015-07-08": 1436313600,
             "2015-07-09": 1436400000,
             "2015-07-10": 1436486400,
             "2015-07-11": 1436572800,
             "2015-07-12": 1436659200,
             "2015-07-13": 1436745600,
             "2015-07-14": 1436832000,
             "2015-07-15": 1436918400,
             "2015-07-16": 1437004800,
             "2015-07-17": 1437091200,
             "2015-07-18": 1437177600,
             "2015-07-19": 1437264000,
             "2015-07-20": 1437350400,
             "2015-07-21": 1437436800,
             "2015-07-22": 1437523200,
             "2015-07-23": 1437609600,
             "2015-07-24": 1437696000,
             "2015-07-25": 1437782400,
             "2015-07-26": 1437868800,
             "2015-07-27": 1437955200,
             "2015-07-28": 1438041600,
             "2015-07-29": 1438128000,
             "2015-07-30": 1438214400,
             "2015-07-31": 1438300800,
             "2015-08-01": 1438387200,
             "2015-08-02": 1438473600,
             "2015-08-03": 1438560000,
             "2015-08-04": 1438646400,
             "2015-08-05": 1438732800,
             "2015-08-06": 1438819200,
             "2015-08-07": 1438905600,
             "2015-08-08": 1438992000,
             "2015-08-09": 1439078400,
             "2015-08-10": 1439164800,
             "2015-08-11": 1439251200,
             "2015-08-12": 1439337600,
             "2015-08-13": 1439424000,
             "2015-08-14": 1439510400,
             "2015-08-15": 1439596800,
             "2015-08-16": 1439683200,
             "2015-08-17": 1439769600,
             "2015-08-18": 1439856000,
             "2015-08-19": 1439942400,
             "2015-08-20": 1440028800,
             "2015-08-21": 1440115200,
             "2015-08-22": 1440201600,
             "2015-08-23": 1440288000,
             "2015-08-24": 1440374400,
             "2015-08-25": 1440460800,
             "2015-08-26": 1440547200,
             "2015-08-27": 1440633600,
             "2015-08-28": 1440720000,
             "2015-08-29": 1440806400,
             "2015-08-30": 1440892800,
             "2015-08-31": 1440979200,
             "2015-09-01": 1441065600,
             "2015-09-02": 1441152000,
             "2015-09-03": 1441238400,
             "2015-09-04": 1441324800,
             "2015-09-05": 1441411200,
             "2015-09-06": 1441497600,
             "2015-09-07": 1441584000,
             "2015-09-08": 1441670400,
             "2015-09-09": 1441756800,
             "2015-09-10": 1441843200,
             "2015-09-11": 1441929600,
             "2015-09-12": 1442016000,
             "2015-09-13": 1442102400,
             "2015-09-14": 1442188800,
             "2015-09-15": 1442275200,
             "2015-09-16": 1442361600,
             "2015-09-17": 1442448000,
             "2015-09-18": 1442534400,
             "2015-09-19": 1442620800,
             "2015-09-20": 1442707200,
             "2015-09-21": 1442793600,
             "2015-09-22": 1442880000,
             "2015-09-23": 1442966400,
             "2015-09-24": 1443052800,
             "2015-09-25": 1443139200,
             "2015-09-26": 1443225600,
             "2015-09-27": 1443312000,
             "2015-09-28": 1443398400,
             "2015-09-29": 1443484800,
             "2015-09-30": 1443571200,
             "2015-10-01": 1443657600,
             "2015-10-02": 1443744000,
             "2015-10-03": 1443830400,
             "2015-10-04": 1443916800,
             "2015-10-05": 1444003200,
             "2015-10-06": 1444089600,
             "2015-10-07": 1444176000,
             "2015-10-08": 1444262400,
             "2015-10-09": 1444348800,
             "2015-10-10": 1444435200,
             "2015-10-11": 1444521600,
             "2015-10-12": 1444608000,
             "2015-10-13": 1444694400,
             "2015-10-14": 1444780800,
             "2015-10-15": 1444867200,
             "2015-10-16": 1444953600,
             "2015-10-17": 1445040000,
             "2015-10-18": 1445126400,
             "2015-10-19": 1445212800,
             "2015-10-20": 1445299200,
             "2015-10-21": 1445385600,
             "2015-10-22": 1445472000,
             "2015-10-23": 1445558400,
             "2015-10-24": 1445644800,
             "2015-10-25": 1445731200,
             "2015-10-26": 1445817600,
             "2015-10-27": 1445904000,
             "2015-10-28": 1445990400,
             "2015-10-29": 1446076800,
             "2015-10-30": 1446163200,
             "2015-10-31": 1446249600,
             "2015-11-01": 1446336000,
             "2015-11-02": 1446422400,
             "2015-11-03": 1446508800,
             "2015-11-04": 1446595200,
             "2015-11-05": 1446681600,
             "2015-11-06": 1446768000,
             "2015-11-07": 1446854400,
             "2015-11-08": 1446940800,
             "2015-11-09": 1447027200,
             "2015-11-10": 1447113600,
             "2015-11-11": 1447200000,
             "2015-11-12": 1447286400,
             "2015-11-13": 1447372800,
             "2015-11-14": 1447459200,
             "2015-11-15": 1447545600,
             "2015-11-16": 1447632000,
             "2015-11-17": 1447718400,
             "2015-11-18": 1447804800,
             "2015-11-19": 1447891200,
             "2015-11-20": 1447977600,
             "2015-11-21": 1448064000,
             "2015-11-22": 1448150400,
             "2015-11-23": 1448236800,
             "2015-11-24": 1448323200,
             "2015-11-25": 1448409600,
             "2015-11-26": 1448496000,
             "2015-11-27": 1448582400,
             "2015-11-28": 1448668800,
             "2015-11-29": 1448755200,
             "2015-11-30": 1448841600,
             "2015-12-01": 1448928000,
             "2015-12-02": 1449014400,
             "2015-12-03": 1449100800,
             "2015-12-04": 1449187200,
             "2015-12-05": 1449273600,
             "2015-12-06": 1449360000,
             "2015-12-07": 1449446400,
             "2015-12-08": 1449532800,
             "2015-12-09": 1449619200,
             "2015-12-10": 1449705600,
             "2015-12-11": 1449792000,
             "2015-12-12": 1449878400,
             "2015-12-13": 1449964800,
             "2015-12-14": 1450051200,
             "2015-12-15": 1450137600,
             "2015-12-16": 1450224000,
             "2015-12-17": 1450310400,
             "2015-12-18": 1450396800,
             "2015-12-19": 1450483200,
             "2015-12-20": 1450569600,
             "2015-12-21": 1450656000,
             "2015-12-22": 1450742400,
             "2015-12-23": 1450828800,
             "2015-12-24": 1450915200,
             "2015-12-25": 1451001600,
             "2015-12-26": 1451088000,
             "2015-12-27": 1451174400,
             "2015-12-28": 1451260800,
             "2015-12-29": 1451347200,
             "2015-12-30": 1451433600,
             "2015-12-31": 1451520000,
             "2016-01-01": 1451606400,
             "2016-01-02": 1451692800,
             "2016-01-03": 1451779200,
             "2016-01-04": 1451865600,
             "2016-01-05": 1451952000,
             "2016-01-06": 1452038400,
             "2016-01-07": 1452124800,
             "2016-01-08": 1452211200,
             "2016-01-09": 1452297600,
             "2016-01-10": 1452384000,
             "2016-01-11": 1452470400,
             "2016-01-12": 1452556800,
             "2016-01-13": 1452643200,
             "2016-01-14": 1452729600,
             "2016-01-15": 1452816000,
             "2016-01-16": 1452902400,
             "2016-01-17": 1452988800,
             "2016-01-18": 1453075200,
             "2016-01-19": 1453161600,
             "2016-01-20": 1453248000,
             "2016-01-21": 1453334400,
             "2016-01-22": 1453420800,
             "2016-01-23": 1453507200,
             "2016-01-24": 1453593600,
             "2016-01-25": 1453680000,
             "2016-01-26": 1453766400,
             "2016-01-27": 1453852800,
             "2016-01-28": 1453939200,
             "2016-01-29": 1454025600,
             "2016-01-30": 1454112000,
             "2016-01-31": 1454198400,
             "2016-02-01": 1454284800,
             "2016-02-02": 1454371200,
             "2016-02-03": 1454457600,
             "2016-02-04": 1454544000,
             "2016-02-05": 1454630400,
             "2016-02-06": 1454716800,
             "2016-02-07": 1454803200,
             "2016-02-08": 1454889600,
             "2016-02-09": 1454976000,
             "2016-02-10": 1455062400,
             "2016-02-11": 1455148800,
             "2016-02-12": 1455235200,
             "2016-02-13": 1455321600,
             "2016-02-14": 1455408000,
             "2016-02-15": 1455494400,
             "2016-02-16": 1455580800,
             "2016-02-17": 1455667200,
             "2016-02-18": 1455753600,
             "2016-02-19": 1455840000,
             "2016-02-20": 1455926400,
             "2016-02-21": 1456012800,
             "2016-02-22": 1456099200,
             "2016-02-23": 1456185600,
             "2016-02-24": 1456272000,
             "2016-02-25": 1456358400,
             "2016-02-26": 1456444800,
             "2016-02-27": 1456531200,
             "2016-02-28": 1456617600,
             "2016-02-29": 1456704000,
             "2016-03-01": 1456790400,
             "2016-03-02": 1456876800,
             "2016-03-03": 1456963200,
             "2016-03-04": 1457049600,
             "2016-03-05": 1457136000,
             "2016-03-06": 1457222400,
             "2016-03-07": 1457308800,
             "2016-03-08": 1457395200,
             "2016-03-09": 1457481600,
             "2016-03-10": 1457568000,
             "2016-03-11": 1457654400,
             "2016-03-12": 1457740800,
             "2016-03-13": 1457827200,
             "2016-03-14": 1457913600,
             "2016-03-15": 1458000000,
             "2016-03-16": 1458086400,
             "2016-03-17": 1458172800,
             "2016-03-18": 1458259200,
             "2016-03-19": 1458345600,
             "2016-03-20": 1458432000,
             "2016-03-21": 1458518400,
             "2016-03-22": 1458604800,
             "2016-03-23": 1458691200,
             "2016-03-24": 1458777600,
             "2016-03-25": 1458864000,
             "2016-03-26": 1458950400,
             "2016-03-27": 1459036800,
             "2016-03-28": 1459123200,
             "2016-03-29": 1459209600,
             "2016-03-30": 1459296000,
             "2016-03-31": 1459382400,
             "2016-04-01": 1459468800,
             "2016-04-02": 1459555200,
             "2016-04-03": 1459641600,
             "2016-04-04": 1459728000,
             "2016-04-05": 1459814400,
             "2016-04-06": 1459900800,
             "2016-04-07": 1459987200,
             "2016-04-08": 1460073600,
             "2016-04-09": 1460160000,
             "2016-04-10": 1460246400,
             "2016-04-11": 1460332800,
             "2016-04-12": 1460419200,
             "2016-04-13": 1460505600,
             "2016-04-14": 1460592000,
             "2016-04-15": 1460678400,
             "2016-04-16": 1460764800,
             "2016-04-17": 1460851200,
             "2016-04-18": 1460937600,
             "2016-04-19": 1461024000,
             "2016-04-20": 1461110400,
             "2016-04-21": 1461196800,
             "2016-04-22": 1461283200,
             "2016-04-23": 1461369600,
             "2016-04-24": 1461456000,
             "2016-04-25": 1461542400,
             "2016-04-26": 1461628800,
             "2016-04-27": 1461715200,
             "2016-04-28": 1461801600,
             "2016-04-29": 1461888000,
             "2016-04-30": 1461974400,
             "2016-05-01": 1462060800,
             "2016-05-02": 1462147200,
             "2016-05-03": 1462233600,
             "2016-05-04": 1462320000,
             "2016-05-05": 1462406400,
             "2016-05-06": 1462492800,
             "2016-05-07": 1462579200,
             "2016-05-08": 1462665600,
             "2016-05-09": 1462752000,
             "2016-05-10": 1462838400,
             "2016-05-11": 1462924800,
             "2016-05-12": 1463011200,
             "2016-05-13": 1463097600,
             "2016-05-14": 1463184000,
             "2016-05-15": 1463270400,
             "2016-05-16": 1463356800,
             "2016-05-17": 1463443200,
             "2016-05-18": 1463529600,
             "2016-05-19": 1463616000,
             "2016-05-20": 1463702400,
             "2016-05-21": 1463788800,
             "2016-05-22": 1463875200,
             "2016-05-23": 1463961600,
             "2016-05-24": 1464048000,
             "2016-05-25": 1464134400,
             "2016-05-26": 1464220800,
             "2016-05-27": 1464307200,
             "2016-05-28": 1464393600,
             "2016-05-29": 1464480000,
             "2016-05-30": 1464566400,
             "2016-05-31": 1464652800,
             "2016-06-01": 1464739200,
             "2016-06-02": 1464825600,
             "2016-06-03": 1464912000,
             "2016-06-04": 1464998400,
             "2016-06-05": 1465084800,
             "2016-06-06": 1465171200,
             "2016-06-07": 1465257600,
             "2016-06-08": 1465344000,
             "2016-06-09": 1465430400,
             "2016-06-10": 1465516800,
             "2016-06-11": 1465603200,
             "2016-06-12": 1465689600,
             "2016-06-13": 1465776000,
             "2016-06-14": 1465862400,
             "2016-06-15": 1465948800,
             "2016-06-16": 1466035200,
             "2016-06-17": 1466121600,
             "2016-06-18": 1466208000,
             "2016-06-19": 1466294400,
             "2016-06-20": 1466380800,
             "2016-06-21": 1466467200,
             "2016-06-22": 1466553600,
             "2016-06-23": 1466640000,
             "2016-06-24": 1466726400,
             "2016-06-25": 1466812800,
             "2016-06-26": 1466899200,
             "2016-06-27": 1466985600,
             "2016-06-28": 1467072000,
             "2016-06-29": 1467158400,
             "2016-06-30": 1467244800,
             "2016-07-01": 1467331200,
             "2016-07-02": 1467417600,
             "2016-07-03": 1467504000,
             "2016-07-04": 1467590400,
             "2016-07-05": 1467676800,
             "2016-07-06": 1467763200,
             "2016-07-07": 1467849600,
             "2016-07-08": 1467936000,
             "2016-07-09": 1468022400,
             "2016-07-10": 1468108800,
             "2016-07-11": 1468195200,
             "2016-07-12": 1468281600,
             "2016-07-13": 1468368000,
             "2016-07-14": 1468454400,
             "2016-07-15": 1468540800,
             "2016-07-16": 1468627200,
             "2016-07-17": 1468713600,
             "2016-07-18": 1468800000,
             "2016-07-19": 1468886400,
             "2016-07-20": 1468972800,
             "2016-07-21": 1469059200,
             "2016-07-22": 1469145600,
             "2016-07-23": 1469232000,
             "2016-07-24": 1469318400,
             "2016-07-25": 1469404800,
             "2016-07-26": 1469491200,
             "2016-07-27": 1469577600,
             "2016-07-28": 1469664000,
             "2016-07-29": 1469750400,
             "2016-07-30": 1469836800,
             "2016-07-31": 1469923200,
             "2016-08-01": 1470009600,
             "2016-08-02": 1470096000,
             "2016-08-03": 1470182400,
             "2016-08-04": 1470268800,
             "2016-08-05": 1470355200,
             "2016-08-06": 1470441600,
             "2016-08-07": 1470528000,
             "2016-08-08": 1470614400,
             "2016-08-09": 1470700800,
             "2016-08-10": 1470787200,
             "2016-08-11": 1470873600,
             "2016-08-12": 1470960000,
             "2016-08-13": 1471046400,
             "2016-08-14": 1471132800,
             "2016-08-15": 1471219200,
             "2016-08-16": 1471305600,
             "2016-08-17": 1471392000,
             "2016-08-18": 1471478400,
             "2016-08-19": 1471564800,
             "2016-08-20": 1471651200,
             "2016-08-21": 1471737600,
             "2016-08-22": 1471824000,
             "2016-08-23": 1471910400,
             "2016-08-24": 1471996800,
             "2016-08-25": 1472083200,
             "2016-08-26": 1472169600,
             "2016-08-27": 1472256000,
             "2016-08-28": 1472342400,
             "2016-08-29": 1472428800,
             "2016-08-30": 1472515200,
             "2016-08-31": 1472601600,
             "2016-09-01": 1472688000,
             "2016-09-02": 1472774400,
             "2016-09-03": 1472860800,
             "2016-09-04": 1472947200,
             "2016-09-05": 1473033600,
             "2016-09-06": 1473120000,
             "2016-09-07": 1473206400,
             "2016-09-08": 1473292800,
             "2016-09-09": 1473379200,
             "2016-09-10": 1473465600,
             "2016-09-11": 1473552000,
             "2016-09-12": 1473638400,
             "2016-09-13": 1473724800,
             "2016-09-14": 1473811200,
             "2016-09-15": 1473897600,
             "2016-09-16": 1473984000,
             "2016-09-17": 1474070400,
             "2016-09-18": 1474156800,
             "2016-09-19": 1474243200,
             "2016-09-20": 1474329600,
             "2016-09-21": 1474416000,
             "2016-09-22": 1474502400,
             "2016-09-23": 1474588800,
             "2016-09-24": 1474675200,
             "2016-09-25": 1474761600,
             "2016-09-26": 1474848000,
             "2016-09-27": 1474934400,
             "2016-09-28": 1475020800,
             "2016-09-29": 1475107200,
             "2016-09-30": 1475193600,
             "2016-10-01": 1475280000,
             "2016-10-02": 1475366400,
             "2016-10-03": 1475452800,
             "2016-10-04": 1475539200,
             "2016-10-05": 1475625600,
             "2016-10-06": 1475712000,
             "2016-10-07": 1475798400,
             "2016-10-08": 1475884800,
             "2016-10-09": 1475971200,
             "2016-10-10": 1476057600,
             "2016-10-11": 1476144000,
             "2016-10-12": 1476230400,
             "2016-10-13": 1476316800,
             "2016-10-14": 1476403200,
             "2016-10-15": 1476489600,
             "2016-10-16": 1476576000,
             "2016-10-17": 1476662400,
             "2016-10-18": 1476748800,
             "2016-10-19": 1476835200,
             "2016-10-20": 1476921600,
             "2016-10-21": 1477008000,
             "2016-10-22": 1477094400,
             "2016-10-23": 1477180800,
             "2016-10-24": 1477267200,
             "2016-10-25": 1477353600,
             "2016-10-26": 1477440000,
             "2016-10-27": 1477526400,
             "2016-10-28": 1477612800,
             "2016-10-29": 1477699200,
             "2016-10-30": 1477785600,
             "2016-10-31": 1477872000,
             "2016-11-01": 1477958400,
             "2016-11-02": 1478044800,
             "2016-11-03": 1478131200,
             "2016-11-04": 1478217600,
             "2016-11-05": 1478304000,
             "2016-11-06": 1478390400,
             "2016-11-07": 1478476800,
             "2016-11-08": 1478563200,
             "2016-11-09": 1478649600,
             "2016-11-10": 1478736000,
             "2016-11-11": 1478822400,
             "2016-11-12": 1478908800,
             "2016-11-13": 1478995200,
             "2016-11-14": 1479081600,
             "2016-11-15": 1479168000,
             "2016-11-16": 1479254400,
             "2016-11-17": 1479340800,
             "2016-11-18": 1479427200,
             "2016-11-19": 1479513600,
             "2016-11-20": 1479600000,
             "2016-11-21": 1479686400,
             "2016-11-22": 1479772800,
             "2016-11-23": 1479859200,
             "2016-11-24": 1479945600,
             "2016-11-25": 1480032000,
             "2016-11-26": 1480118400,
             "2016-11-27": 1480204800,
             "2016-11-28": 1480291200,
             "2016-11-29": 1480377600,
             "2016-11-30": 1480464000,
             "2016-12-01": 1480550400,
             "2016-12-02": 1480636800,
             "2016-12-03": 1480723200,
             "2016-12-04": 1480809600,
             "2016-12-05": 1480896000,
             "2016-12-06": 1480982400,
             "2016-12-07": 1481068800,
             "2016-12-08": 1481155200,
             "2016-12-09": 1481241600,
             "2016-12-10": 1481328000,
             "2016-12-11": 1481414400,
             "2016-12-12": 1481500800,
             "2016-12-13": 1481587200,
             "2016-12-14": 1481673600,
             "2016-12-15": 1481760000,
             "2016-12-16": 1481846400,
             "2016-12-17": 1481932800,
             "2016-12-18": 1482019200,
             "2016-12-19": 1482105600,
             "2016-12-20": 1482192000,
             "2016-12-21": 1482278400,
             "2016-12-22": 1482364800,
             "2016-12-23": 1482451200,
             "2016-12-24": 1482537600,
             "2016-12-25": 1482624000,
             "2016-12-26": 1482710400,
             "2016-12-27": 1482796800,
             "2016-12-28": 1482883200,
             "2016-12-29": 1482969600,
             "2016-12-30": 1483056000,
             "2016-12-31": 1483142400,
             "2017-01-01": 1483228800,
             "2017-01-02": 1483315200,
             "2017-01-03": 1483401600,
             "2017-01-04": 1483488000,
             "2017-01-05": 1483574400,
             "2017-01-06": 1483660800,
             "2017-01-07": 1483747200,
             "2017-01-08": 1483833600,
             "2017-01-09": 1483920000,
             "2017-01-10": 1484006400,
             "2017-01-11": 1484092800,
             "2017-01-12": 1484179200,
             "2017-01-13": 1484265600,
             "2017-01-14": 1484352000,
             "2017-01-15": 1484438400,
             "2017-01-16": 1484524800,
             "2017-01-17": 1484611200,
             "2017-01-18": 1484697600,
             "2017-01-19": 1484784000,
             "2017-01-20": 1484870400,
             "2017-01-21": 1484956800,
             "2017-01-22": 1485043200,
             "2017-01-23": 1485129600,
             "2017-01-24": 1485216000,
             "2017-01-25": 1485302400,
             "2017-01-26": 1485388800,
             "2017-01-27": 1485475200,
             "2017-01-28": 1485561600,
             "2017-01-29": 1485648000,
             "2017-01-30": 1485734400,
             "2017-01-31": 1485820800,
             "2017-02-01": 1485907200,
             "2017-02-02": 1485993600,
             "2017-02-03": 1486080000,
             "2017-02-04": 1486166400,
             "2017-02-05": 1486252800,
             "2017-02-06": 1486339200,
             "2017-02-07": 1486425600,
             "2017-02-08": 1486512000,
             "2017-02-09": 1486598400,
             "2017-02-10": 1486684800,
             "2017-02-11": 1486771200,
             "2017-02-12": 1486857600,
             "2017-02-13": 1486944000,
             "2017-02-14": 1487030400,
             "2017-02-15": 1487116800,
             "2017-02-16": 1487203200,
             "2017-02-17": 1487289600,
             "2017-02-18": 1487376000,
             "2017-02-19": 1487462400,
             "2017-02-20": 1487548800,
             "2017-02-21": 1487635200,
             "2017-02-22": 1487721600,
             "2017-02-23": 1487808000,
             "2017-02-24": 1487894400,
             "2017-02-25": 1487980800,
             "2017-02-26": 1488067200,
             "2017-02-27": 1488153600,
             "2017-02-28": 1488240000,
             "2017-03-01": 1488326400,
             "2017-03-02": 1488412800,
             "2017-03-03": 1488499200,
             "2017-03-04": 1488585600,
             "2017-03-05": 1488672000,
             "2017-03-06": 1488758400,
             "2017-03-07": 1488844800,
             "2017-03-08": 1488931200,
             "2017-03-09": 1489017600,
             "2017-03-10": 1489104000,
             "2017-03-11": 1489190400,
             "2017-03-12": 1489276800,
             "2017-03-13": 1489363200,
             "2017-03-14": 1489449600,
             "2017-03-15": 1489536000,
             "2017-03-16": 1489622400,
             "2017-03-17": 1489708800,
             "2017-03-18": 1489795200,
             "2017-03-19": 1489881600,
             "2017-03-20": 1489968000,
             "2017-03-21": 1490054400,
             "2017-03-22": 1490140800,
             "2017-03-23": 1490227200,
             "2017-03-24": 1490313600,
             "2017-03-25": 1490400000,
             "2017-03-26": 1490486400,
             "2017-03-27": 1490572800,
             "2017-03-28": 1490659200,
             "2017-03-29": 1490745600,
             "2017-03-30": 1490832000,
             "2017-03-31": 1490918400,
             "2017-04-01": 1491004800,
             "2017-04-02": 1491091200,
             "2017-04-03": 1491177600,
             "2017-04-04": 1491264000,
             "2017-04-05": 1491350400,
             "2017-04-06": 1491436800,
             "2017-04-07": 1491523200,
             "2017-04-08": 1491609600,
             "2017-04-09": 1491696000,
             "2017-04-10": 1491782400,
             "2017-04-11": 1491868800,
             "2017-04-12": 1491955200,
             "2017-04-13": 1492041600,
             "2017-04-14": 1492128000,
             "2017-04-15": 1492214400,
             "2017-04-16": 1492300800,
             "2017-04-17": 1492387200,
             "2017-04-18": 1492473600,
             "2017-04-19": 1492560000,
             "2017-04-20": 1492646400,
             "2017-04-21": 1492732800,
             "2017-04-22": 1492819200,
             "2017-04-23": 1492905600,
             "2017-04-24": 1492992000,
             "2017-04-25": 1493078400,
             "2017-04-26": 1493164800,
             "2017-04-27": 1493251200,
             "2017-04-28": 1493337600,
             "2017-04-29": 1493424000,
             "2017-04-30": 1493510400,
             "2017-05-01": 1493596800,
             "2017-05-02": 1493683200,
             "2017-05-03": 1493769600,
             "2017-05-04": 1493856000,
             "2017-05-05": 1493942400,
             "2017-05-06": 1494028800,
             "2017-05-07": 1494115200,
             "2017-05-08": 1494201600,
             "2017-05-09": 1494288000,
             "2017-05-10": 1494374400,
             "2017-05-11": 1494460800,
             "2017-05-12": 1494547200,
             "2017-05-13": 1494633600,
             "2017-05-14": 1494720000,
             "2017-05-15": 1494806400,
             "2017-05-16": 1494892800,
             "2017-05-17": 1494979200,
             "2017-05-18": 1495065600,
             "2017-05-19": 1495152000,
             "2017-05-20": 1495238400,
             "2017-05-21": 1495324800,
             "2017-05-22": 1495411200,
             "2017-05-23": 1495497600,
             "2017-05-24": 1495584000,
             "2017-05-25": 1495670400,
             "2017-05-26": 1495756800,
             "2017-05-27": 1495843200}

# Gross (and incorrect) outliers who should be dealt with manually

banned_hashes=["89092f0324d023dd66890ee725601888", # cannabis 
               "2fbd221392b2a2e62dce6623b6d39af1",
               "3dd5c80e97163b0c5134a282df51245c", 
               "7731adb8dd5243154d6b0d9d1a33020f", # stimulants
               "ea766f428a7b507d8b9e2109f00dc598",
               "57e735f22d52afbd15fa721fc0535772",
               "77ce82283e18482e8d5e37d23e53e458",
               "<KEY>",
               "825f785cacef811bd6419526175bdd36",
               "8a47ee3b6cc00ba4ccef9ae8d8d5d665",
               "<KEY>",
               "620bf1b14902afa2bed31b5e8fcc3ade",
               "c6c007fc698d4bf3e2fb6c6d558def29",
               "<KEY>",
               "<KEY>",
               "d09a5341c72265bcf052179add0ca195",
               "41d5af4be401b3a9630eadf5022566ee",
               "d05a454739cf57a7818565544a92b8b0",
               "33aa8cefc0d9d7711928851ac43c6f2d",
               "061ada457b17494915fadda693f184fc",
               "38ed9ad05302598a9201699a11d50918",
               "38565f8edacf1f5a50397910d17af153",
               "2c3a1ba954a24fb7c3ff66f6fe9b346d", # cocaine
               "3543bb96368b2d06f65cb6b843f0e6d9",
               "bb749d729eef6be98a2e40b072ea4f19", 
               "088c08091d8646ec0366bccd99190a87",
               "9527118f72c8e58ec211690da6ed6e33", # dissociatives
               "8d0c5f11718fd7b00301c4ec26938d1d",
               "dc22e86ccd15f5434f7a9e934dc19b9e",
               "39615654ad83a36b7e73deaa356a8c74",
               "b6876e335e6cf244405199532031d1f5",
               "4f83325bf539a88e6d2b024919aafa08",
               "be081d8b85fd03b63206d2d8026d0093", # hallucinogens
               "444d40bc40531d62a53f64c044f50876", 
               "6033ca8a185d416071650acfa3091c46",
               "3d223e7d9644af9e0b22a522d5070b07",
               "473380ad5448179e310cb7ffa23edd00",
               "a933b3e49e456aad931ee7d87fb6725f",
               "e2b27803161a782c541ad6632c45f222",
               "e349dea108686ac399332397133d6a46", 
               "8b368e452380a986e27bc1ab763baeb9", 
               "3927dd2ffd1ebedddd6017157819ddc1",
               "28c4eed57b7bbf0129e4941a7c02d32f", 
               "<KEY>", 
               "50be248f22e3059e9f58b85a6ce51ad2",
               "a583da68ce195418515b718af9501828", 
               "3fe32acbd943f1ab2c2a7bba83b1b972", 
               "<KEY>", 
               "<KEY>",
               "b12d63b60775bae2b9a8940c3d6810ed",
               "8fd1dc719246c9247f5f85f9951be3e2",
               "dbe0832bccd862b7183356ad670e45f8",
               "26937aa16c79b5c9dfac82050d99d35d",
               "535b32ab819396408ba5d1f0304a6336",
               "274922be6c829161e71d5a6cab719bcc",
               "9ac89f590bc3da0fbee9ec7118f73110",
               "<KEY>",
               "8970ce54e23b3f4f51fdf6ef808556e3",
               "51d28d32e6ed526764ef66bdbd9ee0e5",
               "<KEY>",
               "0dc5ba1c1c88ee03423db962be5887ef",
               "<KEY>",
               "da35990478e5da1b1c376340982c0eb1",
               "<KEY>",
               "<KEY>",
               "<KEY>",
               "10cd02badbd5e2d93d92cf507c602193",
               "3d75921b80ee449434a835673b4f9c0a",
               "d528e50d1881dd6af159308f3b54fcaa",
               "811825c212d57676abd10b51553f0681",
               "46410a9659be3ea5a6c550c0b580e05e", # 2018
               "ce2e255916135720bf4b71919a70a683",
               "25f08dc5443fc779216eb05a3142528e",
               "978dd853b2ddbe429812b685cffc0143",
               "b768ca5876a2aeb945f58e6dd1a4257c"]

manual_hashes = {"0b94b72345828846609b7e44b4890f7d":10.0,
                 "67a712b474d9eb53bd42ab4a6b07bc6c":60.0,
                 "179295d2df6d77824744f71fa3f8372a":0.84,
                 "4714942f575ca381b2493f4ec1788aaf":10.00,
                 "d4da8579427300b2daf909a6683ecdae":10.00,
                 "0328885d8ac53d407b0b59bdbd571f3d":110.00,
                 "9b1d50fe0318dfe75561720a0d616278":17.00,
                 "8f365681b2d2e674c5bbed2359a7c352":11.00,
                 "ea6b9f9aca7853236ffdb991430b583b":80.00,
                 "bd4b3286dc8ac568b18ac2150fac25fa":120.00,
                 "1e36854b7f5d9d4d558f44fe4e2988ff":1100.00,
                 "4ff4fc8870d984ac2ad2945fd444a10e":10.00,
                 "4312f721c4d2bcf0c8a656d820892dfc":11.00,
                 "c593cfc38cefbb25274422c0e29baf66":10.00,
                 "baaab13a0c49e0eda935bfcda8c78f7b":220.00,
                 "8c71e63fa006e16525e15d7ad0e7b987":15.00,
                 "8a10b14a80c272b2933de89299247629":10.00,
                 "fc5b97cd8819bffb72c63ec6616de59c":20.00,
                 "4a1ce4f7e054eb1e64ce273584972999":25.00,
                 "ad4b644de203138cf9e39561471616a6":11.00,
                 "901f6d74210b3afefe65fc25f09a57e0":24.00,
                 "17c69e94007ba2b4b8ba0ab3d171ac7b":18.00,
                 "c6e86b26a655b990b0cf709bbc7beb71":55.00,
                 "7b5851dc1452b1ca52995f596ee5a143":110.00,
                 "528de7500ad667d9e080e81d5abbd01b":10.00,
                 "b8ec704f9257071c1b75d06db0e59dd1":60.00,  
                 "b94f97677ca739702623eacd6689b1d2":0.60,
                 "9762c5f38d713f0bd7e74c01b762bd78":11.00, 
                 "fb5173130773d07131b86e5a5fb2c2bd":40.00, 
                 "1dfc99bf8cb86434f7da2216086f3db8":36.00,
                 "35510b32e5b476e7454a0859eebdc396":750.00,
                 "e0364068d1817c803538eace22e5443c":100.00,  
                 "8860f7ddfea8d1cf9dac7cab5d663899":20.00,
                 "918e47787ae385fafb5faa1a1d6786ee":115.00,
                 "116f0dd1b7a97c70f82748043fcbbae4":44.00, 
                 "c76e31c12ad81c2d3058a378c906e527":100.00,
                 "3f9e3ab539bdddc9a7e6e42352008d43":22.00,
                 "b0e194334c86456075ad4ffef82cb8dc":95.00,
                 "1830db6c6ceb06531d200d5e3bdef4c0":200.00, 
                 "29c92a3686e3f375a243e0c83462a2c9":2.00,
                 "808bc6f870babc81e5179e7e9aacfc06":25.00,
                 "d98db97802d13f851c5dce2169ad0fb3":56.25,
                 "d08bff6fbb5ef7a161b795435e4d5efb":160.00,
                 "58030107785c6cd70cd527af8644b8ac":1.05,
                 "ba781db198217828d4982587bbf96cb5":1.12,
                 "61de6080648d7347faf01ccf67c07ef7":20.00,
                 "31ae103b2d8c6febe29d0fe12e8a8a1b":0.8,
                 "ea7bbea717487ad3efeacbd3860a2b3f":0.6,
                 "9b8f68c8c5c0a22547eabfb64bca163e":100.0,
                 "1b17284983f677115f43b1aa2992e9dd":20.0,
                 "214133d39d04da65676bdfe22d0ee200":28.3,
                 "56ec3feb65493dfef0cfde6f046dbbdd":453.0,
                 "87cf6dd3d503b35dd53738fdf294ff25":28.3,
                 "372321801a8bb9723865b8d7349e473e":453.0,
                 "2ba43da00abf94864276d4da9b531ff4":28.3,
                 "4e9e749986366f257a921da8cb44ec3e":0.22,
                 "73ac5a208d14157e5f076f66c5cc800d":0.25,
                 "b3e3cc37a10f4d12ec4551bc51e8fafe": 0.225,
                 "e1e18d4c1baf58d98509edaff2fcac2b": 0.00110,
                 "7b8cc4d9abc881c406821df2e2bcc682": 0.130,
                 "260c4b5306accacfcd5483ba7c8205e6": 1,
                 "d7603690106aec9cbbac1ff3a141d1c0": 60,
                 "f6778bc62c6bae31c536fe8b0120a711": 56,
                 "2aec9f397c29445cf91fe66926bfab14": 42,
                 "2e24b264f4e08d666c0f91331bc4e7e8": 28,
                 "f687c80838a44026562bba4e8f97ba9a": 35,
                 "4de22b919af2671722cf8ca89463b991": 2,
                 "ac4adb478358dee0e557f5074a5b2580": 100}

manual_qty = {"95614779ba53405061b5283a77ad387b": 1,
              "a4dd809c8c42c3bb46ad9e76665438e8": 1,
              "6ba4fda23a47ec44460c819cbdf941e0": 10,
              "b465207a52a7d536bdf18e5f4309dac3": 1,
              "4f27315a40f56581e5c0376b15d97fb6": 1, 
              "267fbfdc30388d7bc91889db4e2d8a3e": 1,
              "0bfcee554a7a217dc7cb3ef8316207d1": 1,
              "c92f3d6ec895c09690cbc107a68be33b": 1,
              "bc173ac9607f5f8668e2b9db84ef6e48": 1, 
              "3c10973657ee074578273aa0cdd8f266": 1,
              "4ba0dda79faa950e94ba9e552b201c4b": 1,
              "8f1dce121bd0140c46ffa497e1bc4dec": 1,
              "1dcead8c07b65fc39cc482488eff179c": 1,
              "a02e50556c68dd4dbe47a7fff45111ad": 1,
              "70b42ba4bfe7671e3487bf121346924e": 1,
              "8927de4f0f67bf308959d460092960df": 1,
              "c1f05f3095005e03a041904ca5bf36fb": 1,
              "4fd8f74523e7ed1c1278f052f33823f7": 1,
              "1f8bb389ce77baaf9480b0cb360d63fc": 1,
              "a3ee28e2f879e8d29d6ea99f724c8498": 1,
              "28f0b93f3d4299aa1006f23b8bd2a1ef": 1,
              "c8ce29944a9e8805040a140ec2f24d7a": 1,
              "9e775d810f47397ebd59c49659c5c698": 1,
              "efb9832930aa96dea4a7547ed7d425b0": 1,
              "c3c057b34cb3e0e5926dd0ef96622db2": 1,
              "818b92efb6b3945d5cca386a08dca41f": 1,
              "8334a92ef594d377a0423833759f224c": 1,
              "dcc386125956d6104da44f02963a8fe0": 1,
              "b291a25a08615f69c3178a010135930a": 1,
              "243a1ce151d27e4233f50c2453fa60a1": 1,
              "cab54ecd588ace227aa00565fbce12df": 1,
              "33de352a4430a295cba32224b8d0a967": 1,
              "3191341e9f9d5804f1f06f32ae1f0087": 1,
              "e5a490dc7c2c0e45733b10b75d9130ca": 1,
              "16a89329b256e6ab190800c8e47ad533": 1,
              "c9b972ac60c3973fae49b3e9d6d4fed4": 1,
              "9ea82cfd6b83a5d574f16568d295dece": 1,
              "b976b1ca7f9278644558dc30bea0e074": 1,
              "595c8a9df3891816f0523bf05bdde2fb": 1,
              "3fdc50a4b539f7f585b89eb44985dd7d": 1,
              "7a726976662db3a1f908c583ff3aa052": 1,
              "e2e60a8eae40bfc9f14968022b5d176f": 1,
              "d3dddb6ae7100d2086ab4455724c0ae5": 1,
              "33f102cfa7c46619437b0728c585806c": 1,
              "0dab7e7af2bb7626b6b66fe0d8e7177f": 1,
              "a03bbe9dd33ad3ba892db00042f6b3e5": 1,
              "aace5c5811c7b698e995774bc1b138d9": 1,
              "f6778bc62c6bae31c536fe8b0120a711": 1,
              "760cbeaa2403e45c2ecb8ac423049f66": 1, 
              "1b49ae67f3beafe03c87e651a2c94fcb": 1,
              "1134156139aac00fcb0d925c2aaf26c6": 1,
              "2308d3e618a96c8ca7c0ff4e3f29aeca": 1,
              "2e24b264f4e08d666c0f91331bc4e7e8": 1,
              "f687c80838a44026562bba4e8f97ba9a": 1,
              "519c486f33a9462da1b9e329da76390a": 1,
              "0594f818145478c97004400044c01282": 1,
              "841cfaed4487028c0b3cc6dbc129877d": 1,
              "68be21a5bd25131b37e3bbd18c91662e": 1,
              "1b5db60f95db459d828cf4f34f2fc74b": 1,
              "ccc34067c338631439fca22fe0bd3ea8": 1,
              "2672bf8e0b1316c7a40ef714809cce6f": 1,
              "877dda43f1d11959c84a21c4a1bb5d41": 1,
              "3b40bb1edfeb781900ad8e4d9751aa24": 1,
              "5540be8736c3771f1d683b2c2e5f89a2": 1,
              "177eab52f272b084519bc346d65658ac": 1,
              "a02e50556c68dd4dbe47a7fff45111ad": 1,
              "70b42ba4bfe7671e3487bf121346924e": 1,
              "e2dd7fec06c62bc69a2f541047c41fdf": 1,
              "ae9a002a947f7db21727136f9e1743f7": 1,
              "1f8bb389ce77baaf9480b0cb360d63fc": 1,
              "067b1513ad082bf631cae43a3b92e5af": 1,
              "a3ee28e2f879e8d29d6ea99f724c8498": 1,
              "0b180f4c7fd7138c8164926fc6eae355": 1,
              "182a21a4436df04c2022f1b55e7ef052": 1,
              "3c91ceb3cd98caab0eaa9e555e0ac995": 1,
              "44e2d2cb8e008c4b69de50da22967194": 1,
              "e3aedee0756c3fcef918310171c7135f": 1,
              "c832c753432d7e4eeb75365b4f0c0cb8": 1,
              "b291a25a08615f69c3178a010135930a": 1,
              "3191341e9f9d5804f1f06f32ae1f0087": 1,
              "6245d0e8a00f7ef3aa46f0a1aa404f34": 1,
              "09ffecff62cde62617041e8adf6228fc": 1,
              "320ffde1817a9bd71d8119a9acc7ce98": 1,
              "845347e8b5a023dd2e3df9daca41bb29": 1,
              "a61b19ef9eca74df1a22bfc3c36c758d": 1,
              "f4822ce4a914a9fd5f8bb5de5ea993a8": 1,
              "baafe124de0a8c8b154161024999f0e2": 1,
              "0b53a3af7a79d8cd89d8fd753e76e6a9": 1,
              "62050abe4aa1b1c78d40a9e1cff7ceae": 1,
              "1b896b76070549a60cd2927cf1449d66": 1,
              "3fbc5203919cb3281c9282e7c28c9e2e": 1,
              "3cc28c5b84594dcb5edd56efd15d7eff": 1,
              "5088d4474b3740bb44839f29fc26beec": 1,
              "d2e7d545ba32c510cfee1002642e70a9": 1,
              "3a2142d4cc6e7e246f842896a915187f": 1,
              "a9565eb6991166282ef2f1398c43be21": 1,
              "ddd1470b80489f72525cf35fb3ac88df": 1,
              "760cbeaa2403e45c2ecb8ac423049f66": 1,
              "1134156139aac00fcb0d925c2aaf26c6": 1,
              "be89be78bc121147c934d612051bdd4e": 1,
              "d4e67dc0e4ece442c2228b3d7fdd7f91": 1,
              "519c486f33a9462da1b9e329da76390a": 1,
              "841cfaed4487028c0b3cc6dbc129877d": 1,
              "4a4c803f7d9e7ce5fc0e0ead2f4b779b": 1,
              "6ca6aa75d247eaa68a0b36f4ac600a41": 1,
              "f6c71e6a5da38e600470e7d5436151fb": 1,
              "2d614fffb95d7ef1f1380a97ac84ce26": 1
              }

SOLD1 = re.compile("SOLD OUT", re.I)
SOLD2 = re.compile("DO NOT ORDER", re.I)
SOLD3 = re.compile("NOT AVAILABLE", re.I)
G = re.compile("\W?[0-9]*[,.]?[0-9]+\s?Gr?a?m?o?m?e?s?\W", re.I)
MG = re.compile("\W?[0-9]*[,.]?[0-9]+\s?MG\W", re.I)
KG = re.compile("\W?[0-9]*[,.]?[0-9]+\s?KG\W", re.I)
KILO = re.compile("\W?[0-9]*[,.]?[0-9]+\s?kilog?r?a?m?m?e?o?s?\W", re.I)
LB = re.compile("\W?[0-9]*[,.]?[0-9]+\s?lbs?\W", re.I)
POUND = re.compile("\W?[0-9]*[,.]?[0-9]+\s?pounds?\W", re.I)
OZ = re.compile("\W?[0-9]*[,./]?[0-9]+\s?OZ?u?n?c?e?s?\W", re.I)
UG = re.compile("\W?[0-9]+\s?[Uµ]G\W", re.I)
MCG = re.compile("\W?[0-9]+\s?MCG\W", re.I)
#ML = re.compile("\W?[0-9]+\s?ML\W", re.I)

QTYX = re.compile("(\W+|^)[0-9]+\s?(x|( \*))\W", re.I)
XQTY = re.compile("\W+x\s*[0-9]+(\W+|$)", re.I)
HITS = re.compile("\W?[0-9]+\s?hits?(\W|$)", re.I)
STAMPS = re.compile("\W?[0-9]+\s?stamps(\W|$)", re.I)
TABS = re.compile("\W?[0-9]+\s?tabl?e?t?s?(\W|$)", re.I)
CAPS = re.compile("\W?[0-9]+\s?caps(\W|$)", re.I)
PCS = re.compile("\W?[0-9]+\s?pi?e?ce?s?(\W|$)", re.I)
UNIT = re.compile("\W?[0-9]+\s?units?(\W|$)", re.I)
PACK = re.compile("\W?[0-9]+\s?packs?(\W|$)", re.I)
PILL = re.compile("\W?[0-9]+(\s|-)?x?t?c?\s?pills?(\W|$)", re.I)
XTC = re.compile("\W?[0-9]+(\s|-)xtc(\W|$)", re.I)
GENQ = re.compile("(\W+|^)[0-9,.]+(\s|\-)+.*MDMA", re.I)
ANTIGENQ = re.compile("^[0-9,.]+(\s|\-)*m?(g|%)r?a?m?s?(\s|\-|\.)+.*MDMA", re.I)

DEBUG = 0
DEBUG_VERBOSE = DEBUG * 0

"""
extract_weight_qty: 
    takes as input a string (line), 
    returns a pair [weight (in grams), quantity]
    defaults are -1 and 1
"""

def extract_weight_qty(line): 

    sold_out = SOLD1.search(line) or SOLD2.search(line) or SOLD3.search(line)
    weight_g = G.search(line)
    weight_mg = MG.search(line)
    weight_kg = KG.search(line)
    weight_kilo = KILO.search(line)
    weight_lb = LB.search(line)
    weight_pound = POUND.search(line)
    weight_oz = OZ.search(line)
    weight_ug = UG.search(line)
    weight_mcg = MCG.search(line)

#    volume_ml = ML.search(line)

    weight = (weight_g or weight_mg or weight_kg or weight_kilo
              or weight_oz or weight_ug or weight_mcg or weight_lb or weight_pound)

    hits = HITS.search(line)
    stamps = STAMPS.search(line)
    tabs = TABS.search(line)
    caps = CAPS.search(line)
    unit = UNIT.search(line)
    pcs = PCS.search(line)
    qtyx = QTYX.search(line)
    xqty = XQTY.search(line)
    pack = PACK.search(line)
    pill = PILL.search(line)
    xtc = XTC.search(line)
    genq = GENQ.search(line)
    antigenq = ANTIGENQ.search(line)

    qty = (qtyx or xqty or hits or stamps or caps or xtc 
           or tabs or pcs or unit or pack or pill or (genq and not antigenq))

    actual_weight = -1
    actual_qty = 1

    if weight and not sold_out: 
        if weight_g: 
            multiplier_grams = 1
            match = weight_g.group(0)
        elif weight_mg:
            multiplier_grams = 0.001
            match = weight_mg.group(0)
        elif weight_kg:
            multiplier_grams = 1000
            match = weight_kg.group(0)
        elif weight_lb:
            multiplier_grams = 453
            match = weight_lb.group(0)
        elif weight_pound:
            multiplier_grams = 453
            match = weight_pound.group(0)
        elif weight_kilo:
            multiplier_grams = 1000
            match = weight_kilo.group(0)
        elif weight_oz:
            multiplier_grams = 28
            match = weight_oz.group(0)
        elif weight_ug:
            multiplier_grams = 0.000001;
            match = weight_ug.group(0)
        elif weight_mcg:
            multiplier_grams = 0.000001;
            match = weight_mcg.group(0)
            
        if match:     
            try:
                actual_weight_str = re.findall('\d*[.,/]?\d+', match)[0]
                if "," in actual_weight_str:
                    [intgr, dec] = actual_weight_str.split(",") 
                    actual_weight_str = intgr+"."+dec
                    actual_weight = float(actual_weight_str)
                if "/" in actual_weight_str:
                    [num, den] = actual_weight_str.split("/") 
                    actual_weight = float(num)/float(den)
                else:
                    actual_weight = float(actual_weight_str)

            except (ValueError, IndexError):
                actual_weight = -1
        if actual_weight > -1: 
            actual_weight = actual_weight*multiplier_grams

    if qty and not sold_out:
        if pack:
            qmatch = pack.group(0)
        if hits:
            qmatch = hits.group(0)
        elif stamps:
            qmatch = stamps.group(0)
        elif tabs:
            qmatch = tabs.group(0)
        elif caps: 
            qmatch = caps.group(0)
        elif pcs:
            qmatch = pcs.group(0)
        elif unit:
            qmatch = unit.group(0)
        elif pill:
            qmatch = pill.group(0)
        elif xtc:
            qmatch = xtc.group(0)
        elif qtyx:
            qmatch = qtyx.group(0)
        elif xqty:
            qmatch = xqty.group(0)
        elif genq and not antigenq:
            qmatch = genq.group(0)

        if qmatch:
            try:
                actual_qty = int(re.findall('\d*[.,]?\d+', qmatch)[0])
            except (ValueError, IndexError):
                actual_qty = 1
    else:
        actual_qty = 1 # by default

    if DEBUG_VERBOSE: 
        if weight: 
            print("%s MATCHES: %s" % (line.rstrip(), match))
        if qty: 
            print("%s QUANTITY: %s" % (line.rstrip(), qmatch))

    # safety, sometimes for MDMA the quantity and weights are both matched
    # in that case, the quantity is likely incorrect
    if (actual_qty >= 50 and actual_weight >= 50 and actual_qty == actual_weight):
        actual_qty = 1

    if DEBUG:
        print("%s" % line.rstrip())
        if (weight or qty): 
            print("Result: %d x %.6f grams\n" % (actual_qty, actual_weight))
        else:
            print("Result: ***No match***\n")

    return([actual_weight, actual_qty])


<filename>sourceCodeRelease/1_individual/helpers.py
""" 
A bunch of helpers to assist with investigation 
Contains: 
    - mapping from freeform countries to country codes (ship from field)
    - extracting quantities and weight from a given item name
<NAME>, 2016
This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.

"""

import re
import sys

EU_free2fixed = {"A boring central-EU country" : "Other",
                 "A boring middle EU country" : "Other",
                 "A boring middle-EU country" : "Other",
                 "Amsterdam" : "NL", 
                 "Andorra" : "Other", 
                 "Austria" : "AT", 
                 "BELGIUM" : "BE",
                 "Belgium" : "BE",
                 "Belguim" : "BE",
                 "Bulgaria" : "BG", 
                 "Central Europe" : "Other",
                 "Centre Europe" : "Other",
                 "Croatia" : "HR", 
                 "Cyprus" : "CY", 
                 "Czech" : "CZ", 
                 "Czech Rep." : "CZ", 
                 "Czech Repubic" : "CZ", 
                 "Czech Republic" : "CZ", 
                 "Czech Republic (EU)" : "CZ", 
                 "Czech republic" : "CZ", 
                 "DENMARK" : "DK", 
                 "DK" : "DK", 
                 "Denmark" : "DK", 
                 "Denmark / UK (Top #4 Seller)" : "Other", 
                 "Deutschland" : "DE", 
                 "Deutschland/Germany" : "DE", 
                 "E.U" : "Other",
                 "E.U." : "Other",
                 "EU" : "Other",
                 "EU - FR" : "FR", 
                 "EU - NOT NETHERLAND !!" : "Other",
                 "EU - NOT NETHERLANDS !!" : "Other",
                 "EU / FINLAND / FRANCE" : "Other",
                 "EU / RETAILER AGENT" : "Other",
                 "EU in sealed envelope" : "Other",
                 "EU, USA, and Canada" : "Other",
                 "EUR" : "Other",
                 "EUROPE" : "Other",
                 "East-EU" : "Other",
                 "Eastern Europe" : "Other",
                 "England" : "UK",
                 "Estonia" : "EE", 
                 "Eu" : "Other",
                 "Europe" : "Other",
                 "Europe Union" : "Other",
                 "Europe or Depots" : "Other",
                 "Europe, not NL" : "Other",
                 "European" : "Other",
                 "European Country" : "Other",
                 "European Union" : "Other",
                 "European Union (NOT from Netherland)" : "Other",
                 "European Union (NOT from Netherlands)" : "Other",
                 "European union" : "Other",
                 "FIN" : "FI", 
                 "FR" : "FR",
                 "FRANCE" : "FR",
                 "FRANCE FULLZ WITH DOB!" : "FR",
                 "Finland" : "FI", 
                 "Finland, Suomi" : "FI", 
                 "France" : "FR", 
                 "France - EU" : "FR", 
                 "France / French" : "FR", 
                 "France/Europe" : "FR", 
                 "French Guiana" : "FR", 
                 "French Southern Territories" : "FR", 
                 "GER" : "DE", 
                 "GERMANY" : "DE", 
                 "Ger" : "DE", 
                 "German" : "DE", 
                 "Germany" : "DE", 
                 "Germany (Monday and Friday)" : "DE", 
                 "Germany (Monday and Thursday)" : "DE", 
                 "Germany Monday and Thursday" : "DE", 
                 "Germany and Belgium" : "Other", 
                 "Greece" : "GR", 
                 "HOLLAND" : "NL", 
                 "Holland" : "NL", 
                 "Hollandia" : "NL", 
                 "Hungary" : "HU",
                 "ITALY" : "IT", 
                 "Ireland" : "IE", 
                 "Italy" : "IT", 
                 "Latvia" : "LV", 
                 "Lithuania" : "LT", 
                 "Luxembourg" : "LU", 
                 "Malta" : "MT",  
                 "Monaco" : "Other", 
                 "NL" : "NL", 
                 "NL / Belgium" : "Other", 
                 "NL / DE" : "Other", 
                 "Netehrlands" : "NL", 
                 "Netherland" : "NL", 
                 "Netherlands" : "NL", 
                 "Netherlands & Germany" : "Other",
                 "Netherlands / Belgium" : "Other",
                 "Netherlands / Germany" : "Other",
                 "Netherlands and Spain" : "Other",
                 "Netherlands or Germany" : "Other",
                 "Nl" : "NL", 
                 "Norway" : "NO", 
                 "POLAND" : "PL", 
                 "Poland" : "PL", 
                 "Portugal" : "PT", 
                 "Reunion" : "FR", 
                 "Reunion (FR)" : "FR", 
                 "Romania" : "RO", 
                 "San Marino" : "Other", 
                 "SWE" : "SE", 
                 "SWEDEN" : "SE", 
                 "Saint Martin (FR)" : "FR", 
                 "Scandinavia" : "Other", 
                 "Slovakia" : "SK", 
                 "Slovenia" : "SI",
                 "Slovenia ( European union country )" : "SI",
                 "Slovenia - EU" : "SI",
                 "Spain" : "ES",
                 "Spain or UK" : "Other", 
                 "Sweden" : "SE", 
                 "Sweden/Europe" : "SE", 
                 "THE NETHERLANDS" : "NL", 
                 "The Netherlands" : "NL", 
                 "The Nettherlands" : "NL", 
                 "Turkey" : "TR", 
                 "U.K" : "UK",
                 "UK" : "UK",
                 "UK and IRELAND" : "Other",
                 "UK and Ireland" : "Other",
                 "UK or Spain" : "Other",
                 "UNITED KINGDOM" : "UK",
                 "Uk"  : "UK",
                 "United KIngdom" : "UK",
                 "United Kingdom" : "UK",
                 "United kingdom" : "UK",
                 "Untied Kingdom" : "UK",
                 "Western europe" : "Other", 
                 "eu" : "Other", 
                 "europe" : "Other", 
                 "european union" : "Other", 
                 "fin" : "FI",
                 "finland" : "FI",
                 "france" : "FR", 
                 "germany" : "DE", 
                 "holland" : "NL", 
                 "netherlands" : "NL", 
                 "nl" : "NL", 
                 "nlgermany" : "Other", 
                 "the Netherlands" : "NL", 
                 "the netherlands" : "NL", 
                 "uk" : "UK", 
                 "united kingdom" : "UK",
                 "Europe except Scandinavia": "Other", 
                 "Belgium/Netherlands": "Other", 
                 "Berlin, Germany": "DE",
                 "Central Europe": "Other",
                 "Czechia, Slovakia": "Other",
                 "DE,AT,CH,DK": "Other", 
                 "DEUTSCHLAND, GERMANY": "DE",
                 "DEUTSCHLAND, GERMANY, EUROPE": "DE",
                 "Danmark": "DK",
                 "Danmark, Spain": "Other", 
                 "E.U.": "Other", 
                 "ESPAÃA": "ES", 
                 "EU": "Other", 
                 "EU - Poland": "PL",
                 "EUROPE": "Other",
                 "EspaÃ±a": "ES",
                 "Europa": "Other",
                 "Europe": "Other",
                 "Germany + Netherlands": "Other",
                 "Germany + netherlands": "Other",
                 "Germany Netherlands": "Other",
                 "Germany, Austria": "Other",
                 "Germany, Czech Rep., Poland": "Other",
                 "Germany, Netherlands or United Kingdom": "Other",
                 "Germany, Netherlands or United Kingdom ( Your's choice )": "Other",
                 "Germany/Deutschland": "Other",
                 "Holland Belgium Germany": "Other",
                 "Holland, Belgium": "Other",
                 "Milan": "IT", 
                 "NETHERLANDS": "NL",
                 "Netherlands, holland": "NL",
                 "Netherlands/Belgium": "Other", 
                 "Netherlands/Germany": "Other",
                 "SPAIN": "ES", 
                 "SWISS to SWISS only!": "CH",
                 "Slovakia Czechia": "Other", 
                 "Slovakia, Czechia": "Other", 
                 "Swiss to Swiss ONLY": "CH",
                 "Swiss to Swiss ONLY!!!": "CH",
                 "TURKEY": "TR", 
                 "The Netherlands": "NL",
                 "The Netherlands / Germany": "Other",
                 "The netherlands": "NL", 
                 "U.K/PL": "Other",
                 "UK to WW": "UK", 
                 "UK/PL": "Other",
                 "Western Europe": "Other", 
                 "czech republic": "CZ", 
                 "e.u.": "Other", 
                 "eu": "Other", 
                 "europe": "Other", 
                 "france": "FR",
                 "germany": "DE",
                 "netherland": "NL",
                 "netherlands": "NL",
                 "switzerland": "CH",
                 "the Netherlands": "NL",
                 "u.k": "UK",
                 "uk": "UK", 
                 "uk vendor (❣◡❣)": "UK",
                 "united kingdom": "UK", 
                 "united kingdom to worldwide": "UK"
                 }

# because python time handling is... gross wrt timezones, 
# this is precomputed

date2unix = {"2014-12-26": 1419552000,
             "2014-12-27": 1419638400,
             "2014-12-28": 1419724800,
             "2014-12-29": 1419811200,
             "2014-12-30": 1419897600,
             "2014-12-31": 1419984000,
             "2015-01-01": 1420070400,
             "2015-01-02": 1420156800,
             "2015-01-03": 1420243200,
             "2015-01-04": 1420329600,
             "2015-01-05": 1420416000,
             "2015-01-06": 1420502400,
             "2015-01-07": 1420588800,
             "2015-01-08": 1420675200,
             "2015-01-09": 1420761600,
             "2015-01-10": 1420848000,
             "2015-01-11": 1420934400,
             "2015-01-12": 1421020800,
             "2015-01-13": 1421107200,
             "2015-01-14": 1421193600,
             "2015-01-15": 1421280000,
             "2015-01-16": 1421366400,
             "2015-01-17": 1421452800,
             "2015-01-18": 1421539200,
             "2015-01-19": 1421625600,
             "2015-01-20": 1421712000,
             "2015-01-21": 1421798400,
             "2015-01-22": 1421884800,
             "2015-01-23": 1421971200,
             "2015-01-24": 1422057600,
             "2015-01-25": 1422144000,
             "2015-01-26": 1422230400,
             "2015-01-27": 1422316800,
             "2015-01-28": 1422403200,
             "2015-01-29": 1422489600,
             "2015-01-30": 1422576000,
             "2015-01-31": 1422662400,
             "2015-02-01": 1422748800,
             "2015-02-02": 1422835200,
             "2015-02-03": 1422921600,
             "2015-02-04": 1423008000,
             "2015-02-05": 1423094400,
             "2015-02-06": 1423180800,
             "2015-02-07": 1423267200,
             "2015-02-08": 1423353600,
             "2015-02-09": 1423440000,
             "2015-02-10": 1423526400,
             "2015-02-11": 1423612800,
             "2015-02-12": 1423699200,
             "2015-02-13": 1423785600,
             "2015-02-14": 1423872000,
             "2015-02-15": 1423958400,
             "2015-02-16": 1424044800,
             "2015-02-17": 1424131200,
             "2015-02-18": 1424217600,
             "2015-02-19": 1424304000,
             "2015-02-20": 1424390400,
             "2015-02-21": 1424476800,
             "2015-02-22": 1424563200,
             "2015-02-23": 1424649600,
             "2015-02-24": 1424736000,
             "2015-02-25": 1424822400,
             "2015-02-26": 1424908800,
             "2015-02-27": 1424995200,
             "2015-02-28": 1425081600,
             "2015-03-01": 1425168000,
             "2015-03-02": 1425254400,
             "2015-03-03": 1425340800,
             "2015-03-04": 1425427200,
             "2015-03-05": 1425513600,
             "2015-03-06": 1425600000,
             "2015-03-07": 1425686400,
             "2015-03-08": 1425772800,
             "2015-03-09": 1425859200,
             "2015-03-10": 1425945600,
             "2015-03-11": 1426032000,
             "2015-03-12": 1426118400,
             "2015-03-13": 1426204800,
             "2015-03-14": 1426291200,
             "2015-03-15": 1426377600,
             "2015-03-16": 1426464000,
             "2015-03-17": 1426550400,
             "2015-03-18": 1426636800,
             "2015-03-19": 1426723200,
             "2015-03-20": 1426809600,
             "2015-03-21": 1426896000,
             "2015-03-22": 1426982400,
             "2015-03-23": 1427068800,
             "2015-03-24": 1427155200,
             "2015-03-25": 1427241600,
             "2015-03-26": 1427328000,
             "2015-03-27": 1427414400,
             "2015-03-28": 1427500800,
             "2015-03-29": 1427587200,
             "2015-03-30": 1427673600,
             "2015-03-31": 1427760000,
             "2015-04-01": 1427846400,
             "2015-04-02": 1427932800,
             "2015-04-03": 1428019200,
             "2015-04-04": 1428105600,
             "2015-04-05": 1428192000,
             "2015-04-06": 1428278400,
             "2015-04-07": 1428364800,
             "2015-04-08": 1428451200,
             "2015-04-09": 1428537600,
             "2015-04-10": 1428624000,
             "2015-04-11": 1428710400,
             "2015-04-12": 1428796800,
             "2015-04-13": 1428883200,
             "2015-04-14": 1428969600,
             "2015-04-15": 1429056000,
             "2015-04-16": 1429142400,
             "2015-04-17": 1429228800,
             "2015-04-18": 1429315200,
             "2015-04-19": 1429401600,
             "2015-04-20": 1429488000,
             "2015-04-21": 1429574400,
             "2015-04-22": 1429660800,
             "2015-04-23": 1429747200,
             "2015-04-24": 1429833600,
             "2015-04-25": 1429920000,
             "2015-04-26": 1430006400,
             "2015-04-27": 1430092800,
             "2015-04-28": 1430179200,
             "2015-04-29": 1430265600,
             "2015-04-30": 1430352000,
             "2015-05-01": 1430438400,
             "2015-05-02": 1430524800,
             "2015-05-03": 1430611200,
             "2015-05-04": 1430697600,
             "2015-05-05": 1430784000,
             "2015-05-06": 1430870400,
             "2015-05-07": 1430956800,
             "2015-05-08": 1431043200,
             "2015-05-09": 1431129600,
             "2015-05-10": 1431216000,
             "2015-05-11": 1431302400,
             "2015-05-12": 1431388800,
             "2015-05-13": 1431475200,
             "2015-05-14": 1431561600,
             "2015-05-15": 1431648000,
             "2015-05-16": 1431734400,
             "2015-05-17": 1431820800,
             "2015-05-18": 1431907200,
             "2015-05-19": 1431993600,
             "2015-05-20": 1432080000,
             "2015-05-21": 1432166400,
             "2015-05-22": 1432252800,
             "2015-05-23": 1432339200,
             "2015-05-24": 1432425600,
             "2015-05-25": 1432512000,
             "2015-05-26": 1432598400,
             "2015-05-27": 1432684800,
             "2015-05-28": 1432771200,
             "2015-05-29": 1432857600,
             "2015-05-30": 1432944000,
             "2015-05-31": 1433030400,
             "2015-06-01": 1433116800,
             "2015-06-02": 1433203200,
             "2015-06-03": 1433289600,
             "2015-06-04": 1433376000,
             "2015-06-05": 1433462400,
             "2015-06-06": 1433548800,
             "2015-06-07": 1433635200,
             "2015-06-08": 1433721600,
             "2015-06-09": 1433808000,
             "2015-06-10": 1433894400,
             "2015-06-11": 1433980800,
             "2015-06-12": 1434067200,
             "2015-06-13": 1434153600,
             "2015-06-14": 1434240000,
             "2015-06-15": 1434326400,
             "2015-06-16": 1434412800,
             "2015-06-17": 1434499200,
             "2015-06-18": 1434585600,
             "2015-06-19": 1434672000,
             "2015-06-20": 1434758400,
             "2015-06-21": 1434844800,
             "2015-06-22": 1434931200,
             "2015-06-23": 1435017600,
             "2015-06-24": 1435104000,
             "2015-06-25": 1435190400,
             "2015-06-26": 1435276800,
             "2015-06-27": 1435363200,
             "2015-06-28": 1435449600,
             "2015-06-29": 1435536000,
             "2015-06-30": 1435622400,
             "2015-07-01": 1435708800,
             "2015-07-02": 1435795200,
             "2015-07-03": 1435881600,
             "2015-07-04": 1435968000,
             "2015-07-05": 1436054400,
             "2015-07-06": 1436140800,
             "2015-07-07": 1436227200,
             "2015-07-08": 1436313600,
             "2015-07-09": 1436400000,
             "2015-07-10": 1436486400,
             "2015-07-11": 1436572800,
             "2015-07-12": 1436659200,
             "2015-07-13": 1436745600,
             "2015-07-14": 1436832000,
             "2015-07-15": 1436918400,
             "2015-07-16": 1437004800,
             "2015-07-17": 1437091200,
             "2015-07-18": 1437177600,
             "2015-07-19": 1437264000,
             "2015-07-20": 1437350400,
             "2015-07-21": 1437436800,
             "2015-07-22": 1437523200,
             "2015-07-23": 1437609600,
             "2015-07-24": 1437696000,
             "2015-07-25": 1437782400,
             "2015-07-26": 1437868800,
             "2015-07-27": 1437955200,
             "2015-07-28": 1438041600,
             "2015-07-29": 1438128000,
             "2015-07-30": 1438214400,
             "2015-07-31": 1438300800,
             "2015-08-01": 1438387200,
             "2015-08-02": 1438473600,
             "2015-08-03": 1438560000,
             "2015-08-04": 1438646400,
             "2015-08-05": 1438732800,
             "2015-08-06": 1438819200,
             "2015-08-07": 1438905600,
             "2015-08-08": 1438992000,
             "2015-08-09": 1439078400,
             "2015-08-10": 1439164800,
             "2015-08-11": 1439251200,
             "2015-08-12": 1439337600,
             "2015-08-13": 1439424000,
             "2015-08-14": 1439510400,
             "2015-08-15": 1439596800,
             "2015-08-16": 1439683200,
             "2015-08-17": 1439769600,
             "2015-08-18": 1439856000,
             "2015-08-19": 1439942400,
             "2015-08-20": 1440028800,
             "2015-08-21": 1440115200,
             "2015-08-22": 1440201600,
             "2015-08-23": 1440288000,
             "2015-08-24": 1440374400,
             "2015-08-25": 1440460800,
             "2015-08-26": 1440547200,
             "2015-08-27": 1440633600,
             "2015-08-28": 1440720000,
             "2015-08-29": 1440806400,
             "2015-08-30": 1440892800,
             "2015-08-31": 1440979200,
             "2015-09-01": 1441065600,
             "2015-09-02": 1441152000,
             "2015-09-03": 1441238400,
             "2015-09-04": 1441324800,
             "2015-09-05": 1441411200,
             "2015-09-06": 1441497600,
             "2015-09-07": 1441584000,
             "2015-09-08": 1441670400,
             "2015-09-09": 1441756800,
             "2015-09-10": 1441843200,
             "2015-09-11": 1441929600,
             "2015-09-12": 1442016000,
             "2015-09-13": 1442102400,
             "2015-09-14": 1442188800,
             "2015-09-15": 1442275200,
             "2015-09-16": 1442361600,
             "2015-09-17": 1442448000,
             "2015-09-18": 1442534400,
             "2015-09-19": 1442620800,
             "2015-09-20": 1442707200,
             "2015-09-21": 1442793600,
             "2015-09-22": 1442880000,
             "2015-09-23": 1442966400,
             "2015-09-24": 1443052800,
             "2015-09-25": 1443139200,
             "2015-09-26": 1443225600,
             "2015-09-27": 1443312000,
             "2015-09-28": 1443398400,
             "2015-09-29": 1443484800,
             "2015-09-30": 1443571200,
             "2015-10-01": 1443657600,
             "2015-10-02": 1443744000,
             "2015-10-03": 1443830400,
             "2015-10-04": 1443916800,
             "2015-10-05": 1444003200,
             "2015-10-06": 1444089600,
             "2015-10-07": 1444176000,
             "2015-10-08": 1444262400,
             "2015-10-09": 1444348800,
             "2015-10-10": 1444435200,
             "2015-10-11": 1444521600,
             "2015-10-12": 1444608000,
             "2015-10-13": 1444694400,
             "2015-10-14": 1444780800,
             "2015-10-15": 1444867200,
             "2015-10-16": 1444953600,
             "2015-10-17": 1445040000,
             "2015-10-18": 1445126400,
             "2015-10-19": 1445212800,
             "2015-10-20": 1445299200,
             "2015-10-21": 1445385600,
             "2015-10-22": 1445472000,
             "2015-10-23": 1445558400,
             "2015-10-24": 1445644800,
             "2015-10-25": 1445731200,
             "2015-10-26": 1445817600,
             "2015-10-27": 1445904000,
             "2015-10-28": 1445990400,
             "2015-10-29": 1446076800,
             "2015-10-30": 1446163200,
             "2015-10-31": 1446249600,
             "2015-11-01": 1446336000,
             "2015-11-02": 1446422400,
             "2015-11-03": 1446508800,
             "2015-11-04": 1446595200,
             "2015-11-05": 1446681600,
             "2015-11-06": 1446768000,
             "2015-11-07": 1446854400,
             "2015-11-08": 1446940800,
             "2015-11-09": 1447027200,
             "2015-11-10": 1447113600,
             "2015-11-11": 1447200000,
             "2015-11-12": 1447286400,
             "2015-11-13": 1447372800,
             "2015-11-14": 1447459200,
             "2015-11-15": 1447545600,
             "2015-11-16": 1447632000,
             "2015-11-17": 1447718400,
             "2015-11-18": 1447804800,
             "2015-11-19": 1447891200,
             "2015-11-20": 1447977600,
             "2015-11-21": 1448064000,
             "2015-11-22": 1448150400,
             "2015-11-23": 1448236800,
             "2015-11-24": 1448323200,
             "2015-11-25": 1448409600,
             "2015-11-26": 1448496000,
             "2015-11-27": 1448582400,
             "2015-11-28": 1448668800,
             "2015-11-29": 1448755200,
             "2015-11-30": 1448841600,
             "2015-12-01": 1448928000,
             "2015-12-02": 1449014400,
             "2015-12-03": 1449100800,
             "2015-12-04": 1449187200,
             "2015-12-05": 1449273600,
             "2015-12-06": 1449360000,
             "2015-12-07": 1449446400,
             "2015-12-08": 1449532800,
             "2015-12-09": 1449619200,
             "2015-12-10": 1449705600,
             "2015-12-11": 1449792000,
             "2015-12-12": 1449878400,
             "2015-12-13": 1449964800,
             "2015-12-14": 1450051200,
             "2015-12-15": 1450137600,
             "2015-12-16": 1450224000,
             "2015-12-17": 1450310400,
             "2015-12-18": 1450396800,
             "2015-12-19": 1450483200,
             "2015-12-20": 1450569600,
             "2015-12-21": 1450656000,
             "2015-12-22": 1450742400,
             "2015-12-23": 1450828800,
             "2015-12-24": 1450915200,
             "2015-12-25": 1451001600,
             "2015-12-26": 1451088000,
             "2015-12-27": 1451174400,
             "2015-12-28": 1451260800,
             "2015-12-29": 1451347200,
             "2015-12-30": 1451433600,
             "2015-12-31": 1451520000,
             "2016-01-01": 1451606400,
             "2016-01-02": 1451692800,
             "2016-01-03": 1451779200,
             "2016-01-04": 1451865600,
             "2016-01-05": 1451952000,
             "2016-01-06": 1452038400,
             "2016-01-07": 1452124800,
             "2016-01-08": 1452211200,
             "2016-01-09": 1452297600,
             "2016-01-10": 1452384000,
             "2016-01-11": 1452470400,
             "2016-01-12": 1452556800,
             "2016-01-13": 1452643200,
             "2016-01-14": 1452729600,
             "2016-01-15": 1452816000,
             "2016-01-16": 1452902400,
             "2016-01-17": 1452988800,
             "2016-01-18": 1453075200,
             "2016-01-19": 1453161600,
             "2016-01-20": 1453248000,
             "2016-01-21": 1453334400,
             "2016-01-22": 1453420800,
             "2016-01-23": 1453507200,
             "2016-01-24": 1453593600,
             "2016-01-25": 1453680000,
             "2016-01-26": 1453766400,
             "2016-01-27": 1453852800,
             "2016-01-28": 1453939200,
             "2016-01-29": 1454025600,
             "2016-01-30": 1454112000,
             "2016-01-31": 1454198400,
             "2016-02-01": 1454284800,
             "2016-02-02": 1454371200,
             "2016-02-03": 1454457600,
             "2016-02-04": 1454544000,
             "2016-02-05": 1454630400,
             "2016-02-06": 1454716800,
             "2016-02-07": 1454803200,
             "2016-02-08": 1454889600,
             "2016-02-09": 1454976000,
             "2016-02-10": 1455062400,
             "2016-02-11": 1455148800,
             "2016-02-12": 1455235200,
             "2016-02-13": 1455321600,
             "2016-02-14": 1455408000,
             "2016-02-15": 1455494400,
             "2016-02-16": 1455580800,
             "2016-02-17": 1455667200,
             "2016-02-18": 1455753600,
             "2016-02-19": 1455840000,
             "2016-02-20": 1455926400,
             "2016-02-21": 1456012800,
             "2016-02-22": 1456099200,
             "2016-02-23": 1456185600,
             "2016-02-24": 1456272000,
             "2016-02-25": 1456358400,
             "2016-02-26": 1456444800,
             "2016-02-27": 1456531200,
             "2016-02-28": 1456617600,
             "2016-02-29": 1456704000,
             "2016-03-01": 1456790400,
             "2016-03-02": 1456876800,
             "2016-03-03": 1456963200,
             "2016-03-04": 1457049600,
             "2016-03-05": 1457136000,
             "2016-03-06": 1457222400,
             "2016-03-07": 1457308800,
             "2016-03-08": 1457395200,
             "2016-03-09": 1457481600,
             "2016-03-10": 1457568000,
             "2016-03-11": 1457654400,
             "2016-03-12": 1457740800,
             "2016-03-13": 1457827200,
             "2016-03-14": 1457913600,
             "2016-03-15": 1458000000,
             "2016-03-16": 1458086400,
             "2016-03-17": 1458172800,
             "2016-03-18": 1458259200,
             "2016-03-19": 1458345600,
             "2016-03-20": 1458432000,
             "2016-03-21": 1458518400,
             "2016-03-22": 1458604800,
             "2016-03-23": 1458691200,
             "2016-03-24": 1458777600,
             "2016-03-25": 1458864000,
             "2016-03-26": 1458950400,
             "2016-03-27": 1459036800,
             "2016-03-28": 1459123200,
             "2016-03-29": 1459209600,
             "2016-03-30": 1459296000,
             "2016-03-31": 1459382400,
             "2016-04-01": 1459468800,
             "2016-04-02": 1459555200,
             "2016-04-03": 1459641600,
             "2016-04-04": 1459728000,
             "2016-04-05": 1459814400,
             "2016-04-06": 1459900800,
             "2016-04-07": 1459987200,
             "2016-04-08": 1460073600,
             "2016-04-09": 1460160000,
             "2016-04-10": 1460246400,
             "2016-04-11": 1460332800,
             "2016-04-12": 1460419200,
             "2016-04-13": 1460505600,
             "2016-04-14": 1460592000,
             "2016-04-15": 1460678400,
             "2016-04-16": 1460764800,
             "2016-04-17": 1460851200,
             "2016-04-18": 1460937600,
             "2016-04-19": 1461024000,
             "2016-04-20": 1461110400,
             "2016-04-21": 1461196800,
             "2016-04-22": 1461283200,
             "2016-04-23": 1461369600,
             "2016-04-24": 1461456000,
             "2016-04-25": 1461542400,
             "2016-04-26": 1461628800,
             "2016-04-27": 1461715200,
             "2016-04-28": 1461801600,
             "2016-04-29": 1461888000,
             "2016-04-30": 1461974400,
             "2016-05-01": 1462060800,
             "2016-05-02": 1462147200,
             "2016-05-03": 1462233600,
             "2016-05-04": 1462320000,
             "2016-05-05": 1462406400,
             "2016-05-06": 1462492800,
             "2016-05-07": 1462579200,
             "2016-05-08": 1462665600,
             "2016-05-09": 1462752000,
             "2016-05-10": 1462838400,
             "2016-05-11": 1462924800,
             "2016-05-12": 1463011200,
             "2016-05-13": 1463097600,
             "2016-05-14": 1463184000,
             "2016-05-15": 1463270400,
             "2016-05-16": 1463356800,
             "2016-05-17": 1463443200,
             "2016-05-18": 1463529600,
             "2016-05-19": 1463616000,
             "2016-05-20": 1463702400,
             "2016-05-21": 1463788800,
             "2016-05-22": 1463875200,
             "2016-05-23": 1463961600,
             "2016-05-24": 1464048000,
             "2016-05-25": 1464134400,
             "2016-05-26": 1464220800,
             "2016-05-27": 1464307200,
             "2016-05-28": 1464393600,
             "2016-05-29": 1464480000,
             "2016-05-30": 1464566400,
             "2016-05-31": 1464652800,
             "2016-06-01": 1464739200,
             "2016-06-02": 1464825600,
             "2016-06-03": 1464912000,
             "2016-06-04": 1464998400,
             "2016-06-05": 1465084800,
             "2016-06-06": 1465171200,
             "2016-06-07": 1465257600,
             "2016-06-08": 1465344000,
             "2016-06-09": 1465430400,
             "2016-06-10": 1465516800,
             "2016-06-11": 1465603200,
             "2016-06-12": 1465689600,
             "2016-06-13": 1465776000,
             "2016-06-14": 1465862400,
             "2016-06-15": 1465948800,
             "2016-06-16": 1466035200,
             "2016-06-17": 1466121600,
             "2016-06-18": 1466208000,
             "2016-06-19": 1466294400,
             "2016-06-20": 1466380800,
             "2016-06-21": 1466467200,
             "2016-06-22": 1466553600,
             "2016-06-23": 1466640000,
             "2016-06-24": 1466726400,
             "2016-06-25": 1466812800,
             "2016-06-26": 1466899200,
             "2016-06-27": 1466985600,
             "2016-06-28": 1467072000,
             "2016-06-29": 1467158400,
             "2016-06-30": 1467244800,
             "2016-07-01": 1467331200,
             "2016-07-02": 1467417600,
             "2016-07-03": 1467504000,
             "2016-07-04": 1467590400,
             "2016-07-05": 1467676800,
             "2016-07-06": 1467763200,
             "2016-07-07": 1467849600,
             "2016-07-08": 1467936000,
             "2016-07-09": 1468022400,
             "2016-07-10": 1468108800,
             "2016-07-11": 1468195200,
             "2016-07-12": 1468281600,
             "2016-07-13": 1468368000,
             "2016-07-14": 1468454400,
             "2016-07-15": 1468540800,
             "2016-07-16": 1468627200,
             "2016-07-17": 1468713600,
             "2016-07-18": 1468800000,
             "2016-07-19": 1468886400,
             "2016-07-20": 1468972800,
             "2016-07-21": 1469059200,
             "2016-07-22": 1469145600,
             "2016-07-23": 1469232000,
             "2016-07-24": 1469318400,
             "2016-07-25": 1469404800,
             "2016-07-26": 1469491200,
             "2016-07-27": 1469577600,
             "2016-07-28": 1469664000,
             "2016-07-29": 1469750400,
             "2016-07-30": 1469836800,
             "2016-07-31": 1469923200,
             "2016-08-01": 1470009600,
             "2016-08-02": 1470096000,
             "2016-08-03": 1470182400,
             "2016-08-04": 1470268800,
             "2016-08-05": 1470355200,
             "2016-08-06": 1470441600,
             "2016-08-07": 1470528000,
             "2016-08-08": 1470614400,
             "2016-08-09": 1470700800,
             "2016-08-10": 1470787200,
             "2016-08-11": 1470873600,
             "2016-08-12": 1470960000,
             "2016-08-13": 1471046400,
             "2016-08-14": 1471132800,
             "2016-08-15": 1471219200,
             "2016-08-16": 1471305600,
             "2016-08-17": 1471392000,
             "2016-08-18": 1471478400,
             "2016-08-19": 1471564800,
             "2016-08-20": 1471651200,
             "2016-08-21": 1471737600,
             "2016-08-22": 1471824000,
             "2016-08-23": 1471910400,
             "2016-08-24": 1471996800,
             "2016-08-25": 1472083200,
             "2016-08-26": 1472169600,
             "2016-08-27": 1472256000,
             "2016-08-28": 1472342400,
             "2016-08-29": 1472428800,
             "2016-08-30": 1472515200,
             "2016-08-31": 1472601600,
             "2016-09-01": 1472688000,
             "2016-09-02": 1472774400,
             "2016-09-03": 1472860800,
             "2016-09-04": 1472947200,
             "2016-09-05": 1473033600,
             "2016-09-06": 1473120000,
             "2016-09-07": 1473206400,
             "2016-09-08": 1473292800,
             "2016-09-09": 1473379200,
             "2016-09-10": 1473465600,
             "2016-09-11": 1473552000,
             "2016-09-12": 1473638400,
             "2016-09-13": 1473724800,
             "2016-09-14": 1473811200,
             "2016-09-15": 1473897600,
             "2016-09-16": 1473984000,
             "2016-09-17": 1474070400,
             "2016-09-18": 1474156800,
             "2016-09-19": 1474243200,
             "2016-09-20": 1474329600,
             "2016-09-21": 1474416000,
             "2016-09-22": 1474502400,
             "2016-09-23": 1474588800,
             "2016-09-24": 1474675200,
             "2016-09-25": 1474761600,
             "2016-09-26": 1474848000,
             "2016-09-27": 1474934400,
             "2016-09-28": 1475020800,
             "2016-09-29": 1475107200,
             "2016-09-30": 1475193600,
             "2016-10-01": 1475280000,
             "2016-10-02": 1475366400,
             "2016-10-03": 1475452800,
             "2016-10-04": 1475539200,
             "2016-10-05": 1475625600,
             "2016-10-06": 1475712000,
             "2016-10-07": 1475798400,
             "2016-10-08": 1475884800,
             "2016-10-09": 1475971200,
             "2016-10-10": 1476057600,
             "2016-10-11": 1476144000,
             "2016-10-12": 1476230400,
             "2016-10-13": 1476316800,
             "2016-10-14": 1476403200,
             "2016-10-15": 1476489600,
             "2016-10-16": 1476576000,
             "2016-10-17": 1476662400,
             "2016-10-18": 1476748800,
             "2016-10-19": 1476835200,
             "2016-10-20": 1476921600,
             "2016-10-21": 1477008000,
             "2016-10-22": 1477094400,
             "2016-10-23": 1477180800,
             "2016-10-24": 1477267200,
             "2016-10-25": 1477353600,
             "2016-10-26": 1477440000,
             "2016-10-27": 1477526400,
             "2016-10-28": 1477612800,
             "2016-10-29": 1477699200,
             "2016-10-30": 1477785600,
             "2016-10-31": 1477872000,
             "2016-11-01": 1477958400,
             "2016-11-02": 1478044800,
             "2016-11-03": 1478131200,
             "2016-11-04": 1478217600,
             "2016-11-05": 1478304000,
             "2016-11-06": 1478390400,
             "2016-11-07": 1478476800,
             "2016-11-08": 1478563200,
             "2016-11-09": 1478649600,
             "2016-11-10": 1478736000,
             "2016-11-11": 1478822400,
             "2016-11-12": 1478908800,
             "2016-11-13": 1478995200,
             "2016-11-14": 1479081600,
             "2016-11-15": 1479168000,
             "2016-11-16": 1479254400,
             "2016-11-17": 1479340800,
             "2016-11-18": 1479427200,
             "2016-11-19": 1479513600,
             "2016-11-20": 1479600000,
             "2016-11-21": 1479686400,
             "2016-11-22": 1479772800,
             "2016-11-23": 1479859200,
             "2016-11-24": 1479945600,
             "2016-11-25": 1480032000,
             "2016-11-26": 1480118400,
             "2016-11-27": 1480204800,
             "2016-11-28": 1480291200,
             "2016-11-29": 1480377600,
             "2016-11-30": 1480464000,
             "2016-12-01": 1480550400,
             "2016-12-02": 1480636800,
             "2016-12-03": 1480723200,
             "2016-12-04": 1480809600,
             "2016-12-05": 1480896000,
             "2016-12-06": 1480982400,
             "2016-12-07": 1481068800,
             "2016-12-08": 1481155200,
             "2016-12-09": 1481241600,
             "2016-12-10": 1481328000,
             "2016-12-11": 1481414400,
             "2016-12-12": 1481500800,
             "2016-12-13": 1481587200,
             "2016-12-14": 1481673600,
             "2016-12-15": 1481760000,
             "2016-12-16": 1481846400,
             "2016-12-17": 1481932800,
             "2016-12-18": 1482019200,
             "2016-12-19": 1482105600,
             "2016-12-20": 1482192000,
             "2016-12-21": 1482278400,
             "2016-12-22": 1482364800,
             "2016-12-23": 1482451200,
             "2016-12-24": 1482537600,
             "2016-12-25": 1482624000,
             "2016-12-26": 1482710400,
             "2016-12-27": 1482796800,
             "2016-12-28": 1482883200,
             "2016-12-29": 1482969600,
             "2016-12-30": 1483056000,
             "2016-12-31": 1483142400,
             "2017-01-01": 1483228800,
             "2017-01-02": 1483315200,
             "2017-01-03": 1483401600,
             "2017-01-04": 1483488000,
             "2017-01-05": 1483574400,
             "2017-01-06": 1483660800,
             "2017-01-07": 1483747200,
             "2017-01-08": 1483833600,
             "2017-01-09": 1483920000,
             "2017-01-10": 1484006400,
             "2017-01-11": 1484092800,
             "2017-01-12": 1484179200,
             "2017-01-13": 1484265600,
             "2017-01-14": 1484352000,
             "2017-01-15": 1484438400,
             "2017-01-16": 1484524800,
             "2017-01-17": 1484611200,
             "2017-01-18": 1484697600,
             "2017-01-19": 1484784000,
             "2017-01-20": 1484870400,
             "2017-01-21": 1484956800,
             "2017-01-22": 1485043200,
             "2017-01-23": 1485129600,
             "2017-01-24": 1485216000,
             "2017-01-25": 1485302400,
             "2017-01-26": 1485388800,
             "2017-01-27": 1485475200,
             "2017-01-28": 1485561600,
             "2017-01-29": 1485648000,
             "2017-01-30": 1485734400,
             "2017-01-31": 1485820800,
             "2017-02-01": 1485907200,
             "2017-02-02": 1485993600,
             "2017-02-03": 1486080000,
             "2017-02-04": 1486166400,
             "2017-02-05": 1486252800,
             "2017-02-06": 1486339200,
             "2017-02-07": 1486425600,
             "2017-02-08": 1486512000,
             "2017-02-09": 1486598400,
             "2017-02-10": 1486684800,
             "2017-02-11": 1486771200,
             "2017-02-12": 1486857600,
             "2017-02-13": 1486944000,
             "2017-02-14": 1487030400,
             "2017-02-15": 1487116800,
             "2017-02-16": 1487203200,
             "2017-02-17": 1487289600,
             "2017-02-18": 1487376000,
             "2017-02-19": 1487462400,
             "2017-02-20": 1487548800,
             "2017-02-21": 1487635200,
             "2017-02-22": 1487721600,
             "2017-02-23": 1487808000,
             "2017-02-24": 1487894400,
             "2017-02-25": 1487980800,
             "2017-02-26": 1488067200,
             "2017-02-27": 1488153600,
             "2017-02-28": 1488240000,
             "2017-03-01": 1488326400,
             "2017-03-02": 1488412800,
             "2017-03-03": 1488499200,
             "2017-03-04": 1488585600,
             "2017-03-05": 1488672000,
             "2017-03-06": 1488758400,
             "2017-03-07": 1488844800,
             "2017-03-08": 1488931200,
             "2017-03-09": 1489017600,
             "2017-03-10": 1489104000,
             "2017-03-11": 1489190400,
             "2017-03-12": 1489276800,
             "2017-03-13": 1489363200,
             "2017-03-14": 1489449600,
             "2017-03-15": 1489536000,
             "2017-03-16": 1489622400,
             "2017-03-17": 1489708800,
             "2017-03-18": 1489795200,
             "2017-03-19": 1489881600,
             "2017-03-20": 1489968000,
             "2017-03-21": 1490054400,
             "2017-03-22": 1490140800,
             "2017-03-23": 1490227200,
             "2017-03-24": 1490313600,
             "2017-03-25": 1490400000,
             "2017-03-26": 1490486400,
             "2017-03-27": 1490572800,
             "2017-03-28": 1490659200,
             "2017-03-29": 1490745600,
             "2017-03-30": 1490832000,
             "2017-03-31": 1490918400,
             "2017-04-01": 1491004800,
             "2017-04-02": 1491091200,
             "2017-04-03": 1491177600,
             "2017-04-04": 1491264000,
             "2017-04-05": 1491350400,
             "2017-04-06": 1491436800,
             "2017-04-07": 1491523200,
             "2017-04-08": 1491609600,
             "2017-04-09": 1491696000,
             "2017-04-10": 1491782400,
             "2017-04-11": 1491868800,
             "2017-04-12": 1491955200,
             "2017-04-13": 1492041600,
             "2017-04-14": 1492128000,
             "2017-04-15": 1492214400,
             "2017-04-16": 1492300800,
             "2017-04-17": 1492387200,
             "2017-04-18": 1492473600,
             "2017-04-19": 1492560000,
             "2017-04-20": 1492646400,
             "2017-04-21": 1492732800,
             "2017-04-22": 1492819200,
             "2017-04-23": 1492905600,
             "2017-04-24": 1492992000,
             "2017-04-25": 1493078400,
             "2017-04-26": 1493164800,
             "2017-04-27": 1493251200,
             "2017-04-28": 1493337600,
             "2017-04-29": 1493424000,
             "2017-04-30": 1493510400,
             "2017-05-01": 1493596800,
             "2017-05-02": 1493683200,
             "2017-05-03": 1493769600,
             "2017-05-04": 1493856000,
             "2017-05-05": 1493942400,
             "2017-05-06": 1494028800,
             "2017-05-07": 1494115200,
             "2017-05-08": 1494201600,
             "2017-05-09": 1494288000,
             "2017-05-10": 1494374400,
             "2017-05-11": 1494460800,
             "2017-05-12": 1494547200,
             "2017-05-13": 1494633600,
             "2017-05-14": 1494720000,
             "2017-05-15": 1494806400,
             "2017-05-16": 1494892800,
             "2017-05-17": 1494979200,
             "2017-05-18": 1495065600,
             "2017-05-19": 1495152000,
             "2017-05-20": 1495238400,
             "2017-05-21": 1495324800,
             "2017-05-22": 1495411200,
             "2017-05-23": 1495497600,
             "2017-05-24": 1495584000,
             "2017-05-25": 1495670400,
             "2017-05-26": 1495756800,
             "2017-05-27": 1495843200}

# Gross (and incorrect) outliers who should be dealt with manually

banned_hashes=["89092f0324d023dd66890ee725601888", # cannabis 
               "2fbd221392b2a2e62dce6623b6d39af1",
               "3dd5c80e97163b0c5134a282df51245c", 
               "7731adb8dd5243154d6b0d9d1a33020f", # stimulants
               "ea766f428a7b507d8b9e2109f00dc598",
               "57e735f22d52afbd15fa721fc0535772",
               "77ce82283e18482e8d5e37d23e53e458",
               "<KEY>",
               "825f785cacef811bd6419526175bdd36",
               "8a47ee3b6cc00ba4ccef9ae8d8d5d665",
               "<KEY>",
               "620bf1b14902afa2bed31b5e8fcc3ade",
               "c6c007fc698d4bf3e2fb6c6d558def29",
               "<KEY>",
               "<KEY>",
               "d09a5341c72265bcf052179add0ca195",
               "41d5af4be401b3a9630eadf5022566ee",
               "d05a454739cf57a7818565544a92b8b0",
               "33aa8cefc0d9d7711928851ac43c6f2d",
               "061ada457b17494915fadda693f184fc",
               "38ed9ad05302598a9201699a11d50918",
               "38565f8edacf1f5a50397910d17af153",
               "2c3a1ba954a24fb7c3ff66f6fe9b346d", # cocaine
               "3543bb96368b2d06f65cb6b843f0e6d9",
               "bb749d729eef6be98a2e40b072ea4f19", 
               "088c08091d8646ec0366bccd99190a87",
               "9527118f72c8e58ec211690da6ed6e33", # dissociatives
               "8d0c5f11718fd7b00301c4ec26938d1d",
               "dc22e86ccd15f5434f7a9e934dc19b9e",
               "39615654ad83a36b7e73deaa356a8c74",
               "b6876e335e6cf244405199532031d1f5",
               "4f83325bf539a88e6d2b024919aafa08",
               "be081d8b85fd03b63206d2d8026d0093", # hallucinogens
               "444d40bc40531d62a53f64c044f50876", 
               "6033ca8a185d416071650acfa3091c46",
               "3d223e7d9644af9e0b22a522d5070b07",
               "473380ad5448179e310cb7ffa23edd00",
               "a933b3e49e456aad931ee7d87fb6725f",
               "e2b27803161a782c541ad6632c45f222",
               "e349dea108686ac399332397133d6a46", 
               "8b368e452380a986e27bc1ab763baeb9", 
               "3927dd2ffd1ebedddd6017157819ddc1",
               "28c4eed57b7bbf0129e4941a7c02d32f", 
               "<KEY>", 
               "50be248f22e3059e9f58b85a6ce51ad2",
               "a583da68ce195418515b718af9501828", 
               "3fe32acbd943f1ab2c2a7bba83b1b972", 
               "<KEY>", 
               "<KEY>",
               "b12d63b60775bae2b9a8940c3d6810ed",
               "8fd1dc719246c9247f5f85f9951be3e2",
               "dbe0832bccd862b7183356ad670e45f8",
               "26937aa16c79b5c9dfac82050d99d35d",
               "535b32ab819396408ba5d1f0304a6336",
               "274922be6c829161e71d5a6cab719bcc",
               "9ac89f590bc3da0fbee9ec7118f73110",
               "<KEY>",
               "8970ce54e23b3f4f51fdf6ef808556e3",
               "51d28d32e6ed526764ef66bdbd9ee0e5",
               "<KEY>",
               "0dc5ba1c1c88ee03423db962be5887ef",
               "<KEY>",
               "da35990478e5da1b1c376340982c0eb1",
               "<KEY>",
               "<KEY>",
               "<KEY>",
               "10cd02badbd5e2d93d92cf507c602193",
               "3d75921b80ee449434a835673b4f9c0a",
               "d528e50d1881dd6af159308f3b54fcaa",
               "811825c212d57676abd10b51553f0681",
               "46410a9659be3ea5a6c550c0b580e05e", # 2018
               "ce2e255916135720bf4b71919a70a683",
               "25f08dc5443fc779216eb05a3142528e",
               "978dd853b2ddbe429812b685cffc0143",
               "b768ca5876a2aeb945f58e6dd1a4257c"]

manual_hashes = {"0b94b72345828846609b7e44b4890f7d":10.0,
                 "67a712b474d9eb53bd42ab4a6b07bc6c":60.0,
                 "179295d2df6d77824744f71fa3f8372a":0.84,
                 "4714942f575ca381b2493f4ec1788aaf":10.00,
                 "d4da8579427300b2daf909a6683ecdae":10.00,
                 "0328885d8ac53d407b0b59bdbd571f3d":110.00,
                 "9b1d50fe0318dfe75561720a0d616278":17.00,
                 "8f365681b2d2e674c5bbed2359a7c352":11.00,
                 "ea6b9f9aca7853236ffdb991430b583b":80.00,
                 "bd4b3286dc8ac568b18ac2150fac25fa":120.00,
                 "1e36854b7f5d9d4d558f44fe4e2988ff":1100.00,
                 "4ff4fc8870d984ac2ad2945fd444a10e":10.00,
                 "4312f721c4d2bcf0c8a656d820892dfc":11.00,
                 "c593cfc38cefbb25274422c0e29baf66":10.00,
                 "baaab13a0c49e0eda935bfcda8c78f7b":220.00,
                 "8c71e63fa006e16525e15d7ad0e7b987":15.00,
                 "8a10b14a80c272b2933de89299247629":10.00,
                 "fc5b97cd8819bffb72c63ec6616de59c":20.00,
                 "4a1ce4f7e054eb1e64ce273584972999":25.00,
                 "ad4b644de203138cf9e39561471616a6":11.00,
                 "901f6d74210b3afefe65fc25f09a57e0":24.00,
                 "17c69e94007ba2b4b8ba0ab3d171ac7b":18.00,
                 "c6e86b26a655b990b0cf709bbc7beb71":55.00,
                 "7b5851dc1452b1ca52995f596ee5a143":110.00,
                 "528de7500ad667d9e080e81d5abbd01b":10.00,
                 "b8ec704f9257071c1b75d06db0e59dd1":60.00,  
                 "b94f97677ca739702623eacd6689b1d2":0.60,
                 "9762c5f38d713f0bd7e74c01b762bd78":11.00, 
                 "fb5173130773d07131b86e5a5fb2c2bd":40.00, 
                 "1dfc99bf8cb86434f7da2216086f3db8":36.00,
                 "35510b32e5b476e7454a0859eebdc396":750.00,
                 "e0364068d1817c803538eace22e5443c":100.00,  
                 "8860f7ddfea8d1cf9dac7cab5d663899":20.00,
                 "918e47787ae385fafb5faa1a1d6786ee":115.00,
                 "116f0dd1b7a97c70f82748043fcbbae4":44.00, 
                 "c76e31c12ad81c2d3058a378c906e527":100.00,
                 "3f9e3ab539bdddc9a7e6e42352008d43":22.00,
                 "b0e194334c86456075ad4ffef82cb8dc":95.00,
                 "1830db6c6ceb06531d200d5e3bdef4c0":200.00, 
                 "29c92a3686e3f375a243e0c83462a2c9":2.00,
                 "808bc6f870babc81e5179e7e9aacfc06":25.00,
                 "d98db97802d13f851c5dce2169ad0fb3":56.25,
                 "d08bff6fbb5ef7a161b795435e4d5efb":160.00,
                 "58030107785c6cd70cd527af8644b8ac":1.05,
                 "ba781db198217828d4982587bbf96cb5":1.12,
                 "61de6080648d7347faf01ccf67c07ef7":20.00,
                 "31ae103b2d8c6febe29d0fe12e8a8a1b":0.8,
                 "ea7bbea717487ad3efeacbd3860a2b3f":0.6,
                 "9b8f68c8c5c0a22547eabfb64bca163e":100.0,
                 "1b17284983f677115f43b1aa2992e9dd":20.0,
                 "214133d39d04da65676bdfe22d0ee200":28.3,
                 "56ec3feb65493dfef0cfde6f046dbbdd":453.0,
                 "87cf6dd3d503b35dd53738fdf294ff25":28.3,
                 "372321801a8bb9723865b8d7349e473e":453.0,
                 "2ba43da00abf94864276d4da9b531ff4":28.3,
                 "4e9e749986366f257a921da8cb44ec3e":0.22,
                 "73ac5a208d14157e5f076f66c5cc800d":0.25,
                 "b3e3cc37a10f4d12ec4551bc51e8fafe": 0.225,
                 "e1e18d4c1baf58d98509edaff2fcac2b": 0.00110,
                 "7b8cc4d9abc881c406821df2e2bcc682": 0.130,
                 "260c4b5306accacfcd5483ba7c8205e6": 1,
                 "d7603690106aec9cbbac1ff3a141d1c0": 60,
                 "f6778bc62c6bae31c536fe8b0120a711": 56,
                 "2aec9f397c29445cf91fe66926bfab14": 42,
                 "2e24b264f4e08d666c0f91331bc4e7e8": 28,
                 "f687c80838a44026562bba4e8f97ba9a": 35,
                 "4de22b919af2671722cf8ca89463b991": 2,
                 "ac4adb478358dee0e557f5074a5b2580": 100}

manual_qty = {"95614779ba53405061b5283a77ad387b": 1,
              "a4dd809c8c42c3bb46ad9e76665438e8": 1,
              "6ba4fda23a47ec44460c819cbdf941e0": 10,
              "b465207a52a7d536bdf18e5f4309dac3": 1,
              "4f27315a40f56581e5c0376b15d97fb6": 1, 
              "267fbfdc30388d7bc91889db4e2d8a3e": 1,
              "0bfcee554a7a217dc7cb3ef8316207d1": 1,
              "c92f3d6ec895c09690cbc107a68be33b": 1,
              "bc173ac9607f5f8668e2b9db84ef6e48": 1, 
              "3c10973657ee074578273aa0cdd8f266": 1,
              "4ba0dda79faa950e94ba9e552b201c4b": 1,
              "8f1dce121bd0140c46ffa497e1bc4dec": 1,
              "1dcead8c07b65fc39cc482488eff179c": 1,
              "a02e50556c68dd4dbe47a7fff45111ad": 1,
              "70b42ba4bfe7671e3487bf121346924e": 1,
              "8927de4f0f67bf308959d460092960df": 1,
              "c1f05f3095005e03a041904ca5bf36fb": 1,
              "4fd8f74523e7ed1c1278f052f33823f7": 1,
              "1f8bb389ce77baaf9480b0cb360d63fc": 1,
              "a3ee28e2f879e8d29d6ea99f724c8498": 1,
              "28f0b93f3d4299aa1006f23b8bd2a1ef": 1,
              "c8ce29944a9e8805040a140ec2f24d7a": 1,
              "9e775d810f47397ebd59c49659c5c698": 1,
              "efb9832930aa96dea4a7547ed7d425b0": 1,
              "c3c057b34cb3e0e5926dd0ef96622db2": 1,
              "818b92efb6b3945d5cca386a08dca41f": 1,
              "8334a92ef594d377a0423833759f224c": 1,
              "dcc386125956d6104da44f02963a8fe0": 1,
              "b291a25a08615f69c3178a010135930a": 1,
              "243a1ce151d27e4233f50c2453fa60a1": 1,
              "cab54ecd588ace227aa00565fbce12df": 1,
              "33de352a4430a295cba32224b8d0a967": 1,
              "3191341e9f9d5804f1f06f32ae1f0087": 1,
              "e5a490dc7c2c0e45733b10b75d9130ca": 1,
              "16a89329b256e6ab190800c8e47ad533": 1,
              "c9b972ac60c3973fae49b3e9d6d4fed4": 1,
              "9ea82cfd6b83a5d574f16568d295dece": 1,
              "b976b1ca7f9278644558dc30bea0e074": 1,
              "595c8a9df3891816f0523bf05bdde2fb": 1,
              "3fdc50a4b539f7f585b89eb44985dd7d": 1,
              "7a726976662db3a1f908c583ff3aa052": 1,
              "e2e60a8eae40bfc9f14968022b5d176f": 1,
              "d3dddb6ae7100d2086ab4455724c0ae5": 1,
              "33f102cfa7c46619437b0728c585806c": 1,
              "0dab7e7af2bb7626b6b66fe0d8e7177f": 1,
              "a03bbe9dd33ad3ba892db00042f6b3e5": 1,
              "aace5c5811c7b698e995774bc1b138d9": 1,
              "f6778bc62c6bae31c536fe8b0120a711": 1,
              "760cbeaa2403e45c2ecb8ac423049f66": 1, 
              "1b49ae67f3beafe03c87e651a2c94fcb": 1,
              "1134156139aac00fcb0d925c2aaf26c6": 1,
              "2308d3e618a96c8ca7c0ff4e3f29aeca": 1,
              "2e24b264f4e08d666c0f91331bc4e7e8": 1,
              "f687c80838a44026562bba4e8f97ba9a": 1,
              "519c486f33a9462da1b9e329da76390a": 1,
              "0594f818145478c97004400044c01282": 1,
              "841cfaed4487028c0b3cc6dbc129877d": 1,
              "68be21a5bd25131b37e3bbd18c91662e": 1,
              "1b5db60f95db459d828cf4f34f2fc74b": 1,
              "ccc34067c338631439fca22fe0bd3ea8": 1,
              "2672bf8e0b1316c7a40ef714809cce6f": 1,
              "877dda43f1d11959c84a21c4a1bb5d41": 1,
              "3b40bb1edfeb781900ad8e4d9751aa24": 1,
              "5540be8736c3771f1d683b2c2e5f89a2": 1,
              "177eab52f272b084519bc346d65658ac": 1,
              "a02e50556c68dd4dbe47a7fff45111ad": 1,
              "70b42ba4bfe7671e3487bf121346924e": 1,
              "e2dd7fec06c62bc69a2f541047c41fdf": 1,
              "ae9a002a947f7db21727136f9e1743f7": 1,
              "1f8bb389ce77baaf9480b0cb360d63fc": 1,
              "067b1513ad082bf631cae43a3b92e5af": 1,
              "a3ee28e2f879e8d29d6ea99f724c8498": 1,
              "0b180f4c7fd7138c8164926fc6eae355": 1,
              "182a21a4436df04c2022f1b55e7ef052": 1,
              "3c91ceb3cd98caab0eaa9e555e0ac995": 1,
              "44e2d2cb8e008c4b69de50da22967194": 1,
              "e3aedee0756c3fcef918310171c7135f": 1,
              "c832c753432d7e4eeb75365b4f0c0cb8": 1,
              "b291a25a08615f69c3178a010135930a": 1,
              "3191341e9f9d5804f1f06f32ae1f0087": 1,
              "6245d0e8a00f7ef3aa46f0a1aa404f34": 1,
              "09ffecff62cde62617041e8adf6228fc": 1,
              "320ffde1817a9bd71d8119a9acc7ce98": 1,
              "845347e8b5a023dd2e3df9daca41bb29": 1,
              "a61b19ef9eca74df1a22bfc3c36c758d": 1,
              "f4822ce4a914a9fd5f8bb5de5ea993a8": 1,
              "baafe124de0a8c8b154161024999f0e2": 1,
              "0b53a3af7a79d8cd89d8fd753e76e6a9": 1,
              "62050abe4aa1b1c78d40a9e1cff7ceae": 1,
              "1b896b76070549a60cd2927cf1449d66": 1,
              "3fbc5203919cb3281c9282e7c28c9e2e": 1,
              "3cc28c5b84594dcb5edd56efd15d7eff": 1,
              "5088d4474b3740bb44839f29fc26beec": 1,
              "d2e7d545ba32c510cfee1002642e70a9": 1,
              "3a2142d4cc6e7e246f842896a915187f": 1,
              "a9565eb6991166282ef2f1398c43be21": 1,
              "ddd1470b80489f72525cf35fb3ac88df": 1,
              "760cbeaa2403e45c2ecb8ac423049f66": 1,
              "1134156139aac00fcb0d925c2aaf26c6": 1,
              "be89be78bc121147c934d612051bdd4e": 1,
              "d4e67dc0e4ece442c2228b3d7fdd7f91": 1,
              "519c486f33a9462da1b9e329da76390a": 1,
              "841cfaed4487028c0b3cc6dbc129877d": 1,
              "4a4c803f7d9e7ce5fc0e0ead2f4b779b": 1,
              "6ca6aa75d247eaa68a0b36f4ac600a41": 1,
              "f6c71e6a5da38e600470e7d5436151fb": 1,
              "2d614fffb95d7ef1f1380a97ac84ce26": 1
              }

SOLD1 = re.compile("SOLD OUT", re.I)
SOLD2 = re.compile("DO NOT ORDER", re.I)
SOLD3 = re.compile("NOT AVAILABLE", re.I)
G = re.compile("\W?[0-9]*[,.]?[0-9]+\s?Gr?a?m?o?m?e?s?\W", re.I)
MG = re.compile("\W?[0-9]*[,.]?[0-9]+\s?MG\W", re.I)
KG = re.compile("\W?[0-9]*[,.]?[0-9]+\s?KG\W", re.I)
KILO = re.compile("\W?[0-9]*[,.]?[0-9]+\s?kilog?r?a?m?m?e?o?s?\W", re.I)
LB = re.compile("\W?[0-9]*[,.]?[0-9]+\s?lbs?\W", re.I)
POUND = re.compile("\W?[0-9]*[,.]?[0-9]+\s?pounds?\W", re.I)
OZ = re.compile("\W?[0-9]*[,./]?[0-9]+\s?OZ?u?n?c?e?s?\W", re.I)
UG = re.compile("\W?[0-9]+\s?[Uµ]G\W", re.I)
MCG = re.compile("\W?[0-9]+\s?MCG\W", re.I)
#ML = re.compile("\W?[0-9]+\s?ML\W", re.I)

QTYX = re.compile("(\W+|^)[0-9]+\s?(x|( \*))\W", re.I)
XQTY = re.compile("\W+x\s*[0-9]+(\W+|$)", re.I)
HITS = re.compile("\W?[0-9]+\s?hits?(\W|$)", re.I)
STAMPS = re.compile("\W?[0-9]+\s?stamps(\W|$)", re.I)
TABS = re.compile("\W?[0-9]+\s?tabl?e?t?s?(\W|$)", re.I)
CAPS = re.compile("\W?[0-9]+\s?caps(\W|$)", re.I)
PCS = re.compile("\W?[0-9]+\s?pi?e?ce?s?(\W|$)", re.I)
UNIT = re.compile("\W?[0-9]+\s?units?(\W|$)", re.I)
PACK = re.compile("\W?[0-9]+\s?packs?(\W|$)", re.I)
PILL = re.compile("\W?[0-9]+(\s|-)?x?t?c?\s?pills?(\W|$)", re.I)
XTC = re.compile("\W?[0-9]+(\s|-)xtc(\W|$)", re.I)
GENQ = re.compile("(\W+|^)[0-9,.]+(\s|\-)+.*MDMA", re.I)
ANTIGENQ = re.compile("^[0-9,.]+(\s|\-)*m?(g|%)r?a?m?s?(\s|\-|\.)+.*MDMA", re.I)

DEBUG = 0
DEBUG_VERBOSE = DEBUG * 0

"""
extract_weight_qty: 
    takes as input a string (line), 
    returns a pair [weight (in grams), quantity]
    defaults are -1 and 1
"""

def extract_weight_qty(line): 

    sold_out = SOLD1.search(line) or SOLD2.search(line) or SOLD3.search(line)
    weight_g = G.search(line)
    weight_mg = MG.search(line)
    weight_kg = KG.search(line)
    weight_kilo = KILO.search(line)
    weight_lb = LB.search(line)
    weight_pound = POUND.search(line)
    weight_oz = OZ.search(line)
    weight_ug = UG.search(line)
    weight_mcg = MCG.search(line)

#    volume_ml = ML.search(line)

    weight = (weight_g or weight_mg or weight_kg or weight_kilo
              or weight_oz or weight_ug or weight_mcg or weight_lb or weight_pound)

    hits = HITS.search(line)
    stamps = STAMPS.search(line)
    tabs = TABS.search(line)
    caps = CAPS.search(line)
    unit = UNIT.search(line)
    pcs = PCS.search(line)
    qtyx = QTYX.search(line)
    xqty = XQTY.search(line)
    pack = PACK.search(line)
    pill = PILL.search(line)
    xtc = XTC.search(line)
    genq = GENQ.search(line)
    antigenq = ANTIGENQ.search(line)

    qty = (qtyx or xqty or hits or stamps or caps or xtc 
           or tabs or pcs or unit or pack or pill or (genq and not antigenq))

    actual_weight = -1
    actual_qty = 1

    if weight and not sold_out: 
        if weight_g: 
            multiplier_grams = 1
            match = weight_g.group(0)
        elif weight_mg:
            multiplier_grams = 0.001
            match = weight_mg.group(0)
        elif weight_kg:
            multiplier_grams = 1000
            match = weight_kg.group(0)
        elif weight_lb:
            multiplier_grams = 453
            match = weight_lb.group(0)
        elif weight_pound:
            multiplier_grams = 453
            match = weight_pound.group(0)
        elif weight_kilo:
            multiplier_grams = 1000
            match = weight_kilo.group(0)
        elif weight_oz:
            multiplier_grams = 28
            match = weight_oz.group(0)
        elif weight_ug:
            multiplier_grams = 0.000001;
            match = weight_ug.group(0)
        elif weight_mcg:
            multiplier_grams = 0.000001;
            match = weight_mcg.group(0)
            
        if match:     
            try:
                actual_weight_str = re.findall('\d*[.,/]?\d+', match)[0]
                if "," in actual_weight_str:
                    [intgr, dec] = actual_weight_str.split(",") 
                    actual_weight_str = intgr+"."+dec
                    actual_weight = float(actual_weight_str)
                if "/" in actual_weight_str:
                    [num, den] = actual_weight_str.split("/") 
                    actual_weight = float(num)/float(den)
                else:
                    actual_weight = float(actual_weight_str)

            except (ValueError, IndexError):
                actual_weight = -1
        if actual_weight > -1: 
            actual_weight = actual_weight*multiplier_grams

    if qty and not sold_out:
        if pack:
            qmatch = pack.group(0)
        if hits:
            qmatch = hits.group(0)
        elif stamps:
            qmatch = stamps.group(0)
        elif tabs:
            qmatch = tabs.group(0)
        elif caps: 
            qmatch = caps.group(0)
        elif pcs:
            qmatch = pcs.group(0)
        elif unit:
            qmatch = unit.group(0)
        elif pill:
            qmatch = pill.group(0)
        elif xtc:
            qmatch = xtc.group(0)
        elif qtyx:
            qmatch = qtyx.group(0)
        elif xqty:
            qmatch = xqty.group(0)
        elif genq and not antigenq:
            qmatch = genq.group(0)

        if qmatch:
            try:
                actual_qty = int(re.findall('\d*[.,]?\d+', qmatch)[0])
            except (ValueError, IndexError):
                actual_qty = 1
    else:
        actual_qty = 1 # by default

    if DEBUG_VERBOSE: 
        if weight: 
            print("%s MATCHES: %s" % (line.rstrip(), match))
        if qty: 
            print("%s QUANTITY: %s" % (line.rstrip(), qmatch))

    # safety, sometimes for MDMA the quantity and weights are both matched
    # in that case, the quantity is likely incorrect
    if (actual_qty >= 50 and actual_weight >= 50 and actual_qty == actual_weight):
        actual_qty = 1

    if DEBUG:
        print("%s" % line.rstrip())
        if (weight or qty): 
            print("Result: %d x %.6f grams\n" % (actual_qty, actual_weight))
        else:
            print("Result: ***No match***\n")

    return([actual_weight, actual_qty])


A bunch of helpers to assist with investigation  Contains:      - mapping from freeform countries to country codes (ship from field)     - extracting quantities and weight from a given item name <NAME>, 2016 This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. #4 Seller)" : "Other", # because python time handling is... gross wrt timezones, # this is precomputed # Gross (and incorrect) outliers who should be dealt with manually # cannabis # stimulants # cocaine # dissociatives # hallucinogens # 2018 #ML = re.compile("\W?[0-9]+\s?ML\W", re.I) extract_weight_qty:      takes as input a string (line),      returns a pair [weight (in grams), quantity]     defaults are -1 and 1 #    volume_ml = ML.search(line) # by default # safety, sometimes for MDMA the quantity and weights are both matched # in that case, the quantity is likely incorrect