<reponame>svebk/memex-weapons
import sys
import json
import time
import os
import requests
import shutil

#basepath="/srv/skaraman/weapons/"
#website="gunsamerica"

imagecat=json.load(open('imagecat_conf.json','rt'))
timeout_val=2

def mkpath(outpath):
  pos_slash=[pos for pos,c in enumerate(outpath) if c=="/"]
  for pos in pos_slash:
    try:
      os.mkdir(outpath[:pos])
    except:
      pass

def copyFromDump(image_id,out_dir,base_outfn):
  try:
    dump_dir="/srv/skaraman/weapons/JPL_WeaponsImages/image_dump/"
    outpath=os.path.join(out_dir,base_outfn)
    mkpath(outpath)
    shutil.copyfile(dump_dir+image_id, outpath)
    return True
  except Exception as inst:
    print inst
    return False

def getImageFromImagecat(image_id,out_dir,base_outfn):
  url=imagecat['imagecat_host']+"/weapons/alldata/"+str(image_id)
  outpath=os.path.join(out_dir,base_outfn)
  mkpath(outpath)
  try:
    r = requests.get(url, stream=True, auth=(imagecat['user'], imagecat['passwd']), timeout=timeout_val)
    if r.status_code == 200:
      with open(outpath, 'wb') as f:
        r.raw.decode_content = True
        shutil.copyfileobj(r.raw, f)
        return True
  except Exception as inst:
    print "Download failed for img {} that should be saved at {} from url {}.".format(image_id,base_outfn,url)
    print inst 
    return False

def getImageFromOriginalURL(url,out_dir,base_outfn):
  outpath=os.path.join(out_dir,base_outfn)
  mkpath(outpath)
  try:
    r = requests.get(url, stream=True, timeout=timeout_val)
    if r.status_code == 200:
      with open(outpath, 'wb') as f:
        r.raw.decode_content = True
        shutil.copyfileobj(r.raw, f)
        return True
  except Exception as inst:
    print "Download failed for img that should be saved at {} from url {}.".format(base_outfn,url)
    print inst
    return False

def getImg(img,out_dir):
  start_out=len("file:/data2/USCWeaponsStatsGathering/nutch/full_dump/")
  if not os.path.isfile(os.path.join(out_dir,img[0][start_out:])): 
    if not copyFromDump(img[0][start_out:],out_dir,img[0][start_out:]):
      if not getImageFromImagecat(img[0][start_out:],out_dir,img[0][start_out:]):
        if not getImageFromOriginalURL(img[1],out_dir,img[0][start_out:]):
          return False
  return True


if __name__=="__main__":
  if len(sys.argv)!=6:
    print "[Usage] python create_train_list.py ads_cat_list.jsonl extr.jsonl out_train_file.txt out_cats_list_map.txt out_dir"
    quit()

  ads_cat_list=sys.argv[1]
  extr_filename=sys.argv[2]
  out_filename=sys.argv[3]
  out_castlist_filename=sys.argv[4]
  out_dir=sys.argv[5]
  print ads_cat_list,extr_filename,out_filename,out_castlist_filename,out_dir

  # read cat list and store as dict
  ads_cats={}
  all_cats=[]
  with open(ads_cat_list,'rt') as acl:
    for line in acl:
      jl=json.loads(line)
      key=jl.keys()[0]
      cat = "Other"
      #print jl[key]
      tmp_cat=jl[key].strip()
      #time.sleep(1)
      if tmp_cat:
        cat=tmp_cat
      ads_cats[key]=cat
      if cat not in all_cats:
        all_cats.append(cat)

  print len(all_cats),all_cats
  #print ads_cats
  with open(out_castlist_filename,"wt") as ocl:
    for i,cat in enumerate(all_cats):
      ocl.write("{} \"{}\"\n".format(i,cat))
  ocl.close()
  time.sleep(1)

  extr_file=open(extr_filename,'rt')
  out_train=open(out_filename,'wt')
  #extr_file=open("{}{}/extr_{}_ads.jsonl".format(basepath,website,website),"r")
  for line in extr_file:
    one_extr=json.loads(line)
    extrk=one_extr.keys()[0].strip()
    # find corresponding clean category
    if extrk not in ads_cats:
    #print "No category match for {}. Skipping".format(extrk)
        continue
    cat=ads_cats[extrk]       
    # Beware, some price might be auctions prices (e.g. down to 0$...)
    #price=one_extr[extrk]['landmark_extractions']['current_price']
    #condition=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Condition']
    #brand=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Brand']
    #caliber=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Caliber']
    imgs=[x for x in enumerate(one_extr[extrk]['original_doc']['outlinks'])\
         for img in one_extr[extrk]['landmark_extractions']['images'] if x[1].endswith(img)]
    imgs_paths=[(one_extr[extrk]['original_doc']['outpaths'][pos[0]],pos[1]) for pos in imgs]
    # print images and label in train_files
    start_out=len("file:/data2/USCWeaponsStatsGathering/nutch/full_dump/")
    for img in imgs_paths:
      print img[0][start_out:],cat,all_cats.index(cat)
      if getImg(img,out_dir):
        out_train.write("{} {}\n".format(img[0][start_out:],all_cats.index(cat)))
      #time.sleep(2)
  out_train.close()

import sys
import json
import time
import os
import requests
import shutil

#basepath="/srv/skaraman/weapons/"
#website="gunsamerica"

imagecat=json.load(open('imagecat_conf.json','rt'))
timeout_val=2

def mkpath(outpath):
  pos_slash=[pos for pos,c in enumerate(outpath) if c=="/"]
  for pos in pos_slash:
    try:
      os.mkdir(outpath[:pos])
    except:
      pass

def copyFromDump(image_id,out_dir,base_outfn):
  try:
    dump_dir="/srv/skaraman/weapons/JPL_WeaponsImages/image_dump/"
    outpath=os.path.join(out_dir,base_outfn)
    mkpath(outpath)
    shutil.copyfile(dump_dir+image_id, outpath)
    return True
  except Exception as inst:
    print inst
    return False

def getImageFromImagecat(image_id,out_dir,base_outfn):
  url=imagecat['imagecat_host']+"/weapons/alldata/"+str(image_id)
  outpath=os.path.join(out_dir,base_outfn)
  mkpath(outpath)
  try:
    r = requests.get(url, stream=True, auth=(imagecat['user'], imagecat['passwd']), timeout=timeout_val)
    if r.status_code == 200:
      with open(outpath, 'wb') as f:
        r.raw.decode_content = True
        shutil.copyfileobj(r.raw, f)
        return True
  except Exception as inst:
    print "Download failed for img {} that should be saved at {} from url {}.".format(image_id,base_outfn,url)
    print inst 
    return False

def getImageFromOriginalURL(url,out_dir,base_outfn):
  outpath=os.path.join(out_dir,base_outfn)
  mkpath(outpath)
  try:
    r = requests.get(url, stream=True, timeout=timeout_val)
    if r.status_code == 200:
      with open(outpath, 'wb') as f:
        r.raw.decode_content = True
        shutil.copyfileobj(r.raw, f)
        return True
  except Exception as inst:
    print "Download failed for img that should be saved at {} from url {}.".format(base_outfn,url)
    print inst
    return False

def getImg(img,out_dir):
  start_out=len("file:/data2/USCWeaponsStatsGathering/nutch/full_dump/")
  if not os.path.isfile(os.path.join(out_dir,img[0][start_out:])): 
    if not copyFromDump(img[0][start_out:],out_dir,img[0][start_out:]):
      if not getImageFromImagecat(img[0][start_out:],out_dir,img[0][start_out:]):
        if not getImageFromOriginalURL(img[1],out_dir,img[0][start_out:]):
          return False
  return True


if __name__=="__main__":
  if len(sys.argv)!=6:
    print "[Usage] python create_train_list.py ads_cat_list.jsonl extr.jsonl out_train_file.txt out_cats_list_map.txt out_dir"
    quit()

  ads_cat_list=sys.argv[1]
  extr_filename=sys.argv[2]
  out_filename=sys.argv[3]
  out_castlist_filename=sys.argv[4]
  out_dir=sys.argv[5]
  print ads_cat_list,extr_filename,out_filename,out_castlist_filename,out_dir

  # read cat list and store as dict
  ads_cats={}
  all_cats=[]
  with open(ads_cat_list,'rt') as acl:
    for line in acl:
      jl=json.loads(line)
      key=jl.keys()[0]
      cat = "Other"
      #print jl[key]
      tmp_cat=jl[key].strip()
      #time.sleep(1)
      if tmp_cat:
        cat=tmp_cat
      ads_cats[key]=cat
      if cat not in all_cats:
        all_cats.append(cat)

  print len(all_cats),all_cats
  #print ads_cats
  with open(out_castlist_filename,"wt") as ocl:
    for i,cat in enumerate(all_cats):
      ocl.write("{} \"{}\"\n".format(i,cat))
  ocl.close()
  time.sleep(1)

  extr_file=open(extr_filename,'rt')
  out_train=open(out_filename,'wt')
  #extr_file=open("{}{}/extr_{}_ads.jsonl".format(basepath,website,website),"r")
  for line in extr_file:
    one_extr=json.loads(line)
    extrk=one_extr.keys()[0].strip()
    # find corresponding clean category
    if extrk not in ads_cats:
    #print "No category match for {}. Skipping".format(extrk)
        continue
    cat=ads_cats[extrk]       
    # Beware, some price might be auctions prices (e.g. down to 0$...)
    #price=one_extr[extrk]['landmark_extractions']['current_price']
    #condition=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Condition']
    #brand=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Brand']
    #caliber=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Caliber']
    imgs=[x for x in enumerate(one_extr[extrk]['original_doc']['outlinks'])\
         for img in one_extr[extrk]['landmark_extractions']['images'] if x[1].endswith(img)]
    imgs_paths=[(one_extr[extrk]['original_doc']['outpaths'][pos[0]],pos[1]) for pos in imgs]
    # print images and label in train_files
    start_out=len("file:/data2/USCWeaponsStatsGathering/nutch/full_dump/")
    for img in imgs_paths:
      print img[0][start_out:],cat,all_cats.index(cat)
      if getImg(img,out_dir):
        out_train.write("{} {}\n".format(img[0][start_out:],all_cats.index(cat)))
      #time.sleep(2)
  out_train.close()
#basepath="/srv/skaraman/weapons/" #website="gunsamerica" # read cat list and store as dict #print jl[key] #time.sleep(1) #print ads_cats #extr_file=open("{}{}/extr_{}_ads.jsonl".format(basepath,website,website),"r") # find corresponding clean category #print "No category match for {}. Skipping".format(extrk) # Beware, some price might be auctions prices (e.g. down to 0$...) #price=one_extr[extrk]['landmark_extractions']['current_price'] #condition=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Condition'] #brand=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Brand'] #caliber=[one_extr[extrk]['landmark_extractions']['details'][ii]['value'] for ii in range(len(one_extr[extrk]['landmark_extractions']['details'])) if one_extr[extrk]['landmark_extractions']['details'][ii]['label']=='Caliber'] # print images and label in train_files #time.sleep(2)