def inspect_partitions(bucket):
    """Discover the partitions on a bucket via introspection.

    For large buckets which lack s3 inventories, salactus will attempt
    to process objects in parallel on the bucket by breaking the bucket
    into a separate keyspace partitions. It does this with a heurestic
    that attempts to sample the keyspace and determine appropriate subparts.

    This command provides additional visibility into the partitioning of
    a bucket by showing how salactus would partition a given bucket.
    """

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(level=logging.WARNING)

    state = db.db()
    # add db.bucket accessor
    found = None
    for b in state.buckets():
        if b.name == bucket:
            found = b
            break
    if not found:
        click.echo("no bucket named: %s" % bucket)
        return

    keyset = []
    partitions = []

    def process_keyset(bid, page):
        keyset.append(len(page))

    def process_bucket_iterator(bid, prefix, delimiter="", **continuation):
        partitions.append(prefix)

    # synchronous execution
    def invoke(f, *args, **kw):
        return f(*args, **kw)

    # unleash the monkies ;-)
    worker.connection.hincrby = lambda x, y, z: True
    worker.invoke = invoke
    worker.process_keyset = process_keyset
    worker.process_bucket_iterator = process_bucket_iterator

    # kick it off
    worker.process_bucket_partitions(b.bucket_id)

    keys_scanned = sum(keyset)
    click.echo(
        "Found %d partitions %s keys scanned during partitioning" % (
            len(partitions), keys_scanned))
    click.echo("\n".join(partitions))
def inspect_partitions ( bucket ) : logging . basicConfig ( level = logging . INFO , format = "%(asctime)s: %(name)s:%(levelname)s %(message)s" ) logging . getLogger ( 'botocore' ) . setLevel ( level = logging . WARNING ) state = db . db ( ) found = None for b in state . buckets ( ) : if b . name == bucket : found = b break if not found : click . echo ( "no bucket named: %s" % bucket ) return keyset = [ ] partitions = [ ] def process_keyset ( bid , page ) : keyset . append ( len ( page ) ) def process_bucket_iterator ( bid , prefix , delimiter = "" , ** continuation ) : partitions . append ( prefix ) def invoke ( f , * args , ** kw ) : return f ( * args , ** kw ) worker . connection . hincrby = lambda x , y , z : True worker . invoke = invoke worker . process_keyset = process_keyset worker . process_bucket_iterator = process_bucket_iterator worker . process_bucket_partitions ( b . bucket_id ) keys_scanned = sum ( keyset ) click . echo ( "Found %d partitions %s keys scanned during partitioning" % ( len ( partitions ) , keys_scanned ) ) click . echo ( "\n" . join ( partitions ) )
Discover the partitions on a bucket via introspection.

    For large buckets which lack s3 inventories, salactus will attempt
    to process objects in parallel on the bucket by breaking the bucket
    into a separate keyspace partitions. It does this with a heurestic
    that attempts to sample the keyspace and determine appropriate subparts.

    This command provides additional visibility into the partitioning of
    a bucket by showing how salactus would partition a given bucket.