@cli.command()
@click.option('-where', type=str,
              help='An SQL-like query to run alongside the spatial constraint.')
@click.option('-bbox', type=str,
              help='A bounding box describing the area to confine the query to. If not supplied the whole data set is considered.')
@click.option('--debug/--no-debug', default=False,
              help='Toggles debugging mode to write a logfile of the raw /features/list URIs to. Defaults to not.')
@click.option('--bbox-cache/--no-bbox-cache', default=True,
              help='Toggles whether to cache bounding boxes on first run. Defaults to caching.')
@click.option('--minimum-qps', default=10,
              help='The minimum QPS (Queries/Second) to attain on queryable areas. Defaults to 10.')
@click.option('--num-processes', default=10,
              help='The number of concurrent threads to run whilst querying. Defaults to 10 and is loosely correlated wit --minimum-qps.')
@click.argument('table-id', type=str)
@click.argument('outfile', type=click.File(mode='w'))
@pass_context
def list(ctx, where, bbox, debug, bbox_cache, minimum_qps, num_processes, table_id, outfile):

  # Retrieve the primary key column
  table = ctx.service().tables().get(id=table_id, fields="schema,bbox").execute()
  pkey = table['schema']['primaryKey']

  # Fetch bounding boxes
  bboxes = None
  bbox_cache_file = os.path.splitext(outfile.name)[0] + "-bbox-cache.json"

  if bbox is None:
    bbox = table['bbox']
  else:
    bbox = [float(i) for i in bbox.split(",")]

  if bbox_cache and os.path.isfile(bbox_cache_file):
    ctx.log("Retrieving cached bounding boxes...")
    with open(bbox_cache_file) as f:
      bbox_cache_json = json.load(f)

    if bbox_cache_json["bbox"] == bbox:
      bboxes = bbox_cache_json["viable_bboxes"]
    else:
      ctx.log("Cache mismatch for %s, skipping cache." % (bbox_cache_file))

  if bboxes is None:
    ctx.log("Calculating viable bounding boxes (this can take awhile)...")
    bboxes = get_viable_bboxes(ctx, table_id, minimum_qps, bbox, pkey)

    if bbox_cache:
      with open(bbox_cache_file, "w") as f:
        json.dump({
          "bbox": bbox,
          "viable_bboxes": bboxes
        }, f)
  ctx.log("Querying %s bounding boxes..." % (len(bboxes)))

  # Init for multiprocessing
  manager = Manager()
  feature_store = manager.dict()
  debug_store = manager.list()

  # Chunk up the bounding boxes
  chunk_size = int(ceil(len(bboxes) / float(num_processes)))
  if chunk_size < num_processes:
    chunk_size = 1
  chunks = [(
    ctx, bboxes[i:i+chunk_size], where, table_id,
    feature_store, pkey,
    debug, debug_store,
    minimum_qps, num_processes / minimum_qps
  ) for i in range(0, len(bboxes), chunk_size)]

  # Loot All Of The Things!
  start_time = time.time()

  pool = multiprocessing.Pool(processes=num_processes)
  pool.map(get_features, chunks)
  pool.close()
  pool.join()

  elapsed_secs = time.time() - start_time

  # Write features to disk as a GeoJSON blob
  start = time.time()
  features = {
    "type": "FeatureCollection",
    "features": [feature_store[key] for key in feature_store.keys()]
  }
  ctx.log("JSON formatted in %ss" %(round(time.time() - start, 2)))
  start = time.time()
  json.dump(features, outfile)
  ctx.log("JSON written in %ss" %(round(time.time() - start, 2)))

  # Dump debug information to CSV
  if debug:
    stats = tablib.Dataset(headers=('response_code', 'date', 'feature_count', 'time_to_first_byte_secs', 'time_to_last_byte_secs', 'bbox', 'page_num', 'request'))
    for v in debug_store:
      stats.append(v)
    with open(os.path.splitext(outfile.name)[0] + "-debug.csv", "w") as f:
      f.write(stats.csv)

  ctx.log("Got %s features in %smins" % (len(features["features"]), round(elapsed_secs / 60, 2)))
Retrieve features from a vector table.
dataset/ETHPy150Open keithmoss/Hodor/hodor/commands/cmd_features.py/list