def step(self, path=None, peer_table=None):
        """
        Run one step of this algorithm:
        * find the set of missing zonefiles
        * try to fetch each of them
        * store them
        * update our zonefile database

        Fetch rarest zonefiles first, but batch
        whenever possible.

        Return the number of zonefiles fetched
        """

        # if os.environ.get("BLOCKSTACK_TEST", None) == "1":
        #    log.debug("%s: %s step" % (self.hostport, self.__class__.__name__))

        if path is None:
            path = self.atlasdb_path

        num_fetched = 0
        missing_zinfo = None
        peer_hostports = None

        with AtlasPeerTableLocked(peer_table) as ptbl: 
            missing_zfinfo = atlas_find_missing_zonefile_availability( peer_table=ptbl, path=path )
            peer_hostports = ptbl.keys()[:]

        # ask for zonefiles in rarest-first order
        zonefile_ranking = [ (missing_zfinfo[zfhash]['popularity'], zfhash) for zfhash in missing_zfinfo.keys() ]
        zonefile_ranking.sort()
        zonefile_hashes = list(set([zfhash for (_, zfhash) in zonefile_ranking]))
        zonefile_names = dict([(zfhash, missing_zfinfo[zfhash]['names']) for zfhash in zonefile_hashes])
        zonefile_txids = dict([(zfhash, missing_zfinfo[zfhash]['txid']) for zfhash in zonefile_hashes])
        zonefile_block_heights = dict([(zfhash, missing_zfinfo[zfhash]['block_heights']) for zfhash in zonefile_hashes])
        zonefile_origins = self.find_zonefile_origins( missing_zfinfo, peer_hostports )

        # filter out the ones that are already cached
        for i in xrange(0, len(zonefile_hashes)):
            # is this zonefile already cached?
            zfhash = zonefile_hashes[i]
            present = is_zonefile_cached( zfhash, self.zonefile_dir, validate=True )
            if present:
                log.debug("%s: zonefile %s already cached.  Marking present" % (self.hostport, zfhash))
                zonefile_hashes[i] = None

                # mark it as present
                self.set_zonefile_present(zfhash, min(zonefile_block_heights[zfhash]), path=path)

        zonefile_hashes = filter( lambda zfh: zfh is not None, zonefile_hashes )

        if len(zonefile_hashes) > 0:
            log.debug("%s: missing %s unique zonefiles" % (self.hostport, len(zonefile_hashes)))
        
        while len(zonefile_hashes) > 0 and self.running:

            zfhash = zonefile_hashes[0]
            zfnames = zonefile_names[zfhash]
            zftxid = zonefile_txids[zfhash]
            peers = missing_zfinfo[zfhash]['peers']
           
            # try this zonefile's hosts in order by perceived availability
            peers = atlas_rank_peers_by_health( peer_list=peers, with_zero_requests=True )
            if len(peers) > 0:
                log.debug("%s: zonefile %s available from %s peers (%s...)" % (self.hostport, zfhash, len(peers), ",".join(peers[:min(5, len(peers))])))

            for peer_hostport in peers:

                if zfhash not in zonefile_origins[peer_hostport]:
                    # not available
                    log.debug("%s not available from %s" % (zfhash, peer_hostport))
                    continue

                # what other zonefiles can we get?
                # only ask for the ones we don't have
                peer_zonefile_hashes = []
                for zfh in zonefile_origins[peer_hostport]:
                    if zfh in zonefile_hashes:
                        # can ask for this one too
                        peer_zonefile_hashes.append( zfh )

                if len(peer_zonefile_hashes) == 0:
                    log.debug("%s: No zonefiles available from %s" % (self.hostport, peer_hostport))
                    continue

                # get them all
                log.debug("%s: get %s zonefiles from %s" % (self.hostport, len(peer_zonefile_hashes), peer_hostport))
                zonefiles = atlas_get_zonefiles( self.hostport, peer_hostport, peer_zonefile_hashes, peer_table=peer_table )
                if zonefiles is not None:

                    # got zonefiles!
                    stored_zfhashes = self.store_zonefiles( zonefile_names, zonefiles, zonefile_txids, zonefile_block_heights, peer_zonefile_hashes, peer_hostport, path )
                    
                    # don't ask again
                    log.debug("Stored %s zonefiles" % len(stored_zfhashes))
                    for zfh in stored_zfhashes:
                        if zfh in peer_zonefile_hashes:
                            peer_zonefile_hashes.remove(zfh)
                        if zfh in zonefile_hashes:
                            zonefile_hashes.remove(zfh)

                        num_fetched += 1
                
                else:
                    log.debug("%s: no data received from %s" % (self.hostport, peer_hostport))

                with AtlasPeerTableLocked() as ptbl:
                    # if the node didn't actually have these zonefiles, then 
                    # update their inventories so we don't ask for them again.
                    # TODO: do NOT ban nodes that repeatedly lie to us, since the "node" could be a load-balancer for a set of nodes that may or may not have the zonefile
                    for zfh in peer_zonefile_hashes:
                        log.debug("%s: %s did not have %s" % (self.hostport, peer_hostport, zfh))
                        atlas_peer_set_zonefile_status( peer_hostport, zfh, False, zonefile_bits=missing_zfinfo[zfh]['indexes'], peer_table=ptbl )

                        if zfh in zonefile_origins[peer_hostport]:
                            zonefile_origins[peer_hostport].remove( zfh )


            # done with this zonefile
            if zfhash in zonefile_hashes:
                zonefile_hashes.remove(zfhash)

        if len(zonefile_hashes) > 0 or num_fetched > 0:
            log.debug("%s: fetched %s zonefiles" % (self.hostport, num_fetched))

        return num_fetched
def step ( self , path = None , peer_table = None ) : if path is None : path = self . atlasdb_path num_fetched = 0 missing_zinfo = None peer_hostports = None with AtlasPeerTableLocked ( peer_table ) as ptbl : missing_zfinfo = atlas_find_missing_zonefile_availability ( peer_table = ptbl , path = path ) peer_hostports = ptbl . keys ( ) [ : ] zonefile_ranking = [ ( missing_zfinfo [ zfhash ] [ 'popularity' ] , zfhash ) for zfhash in missing_zfinfo . keys ( ) ] zonefile_ranking . sort ( ) zonefile_hashes = list ( set ( [ zfhash for ( _ , zfhash ) in zonefile_ranking ] ) ) zonefile_names = dict ( [ ( zfhash , missing_zfinfo [ zfhash ] [ 'names' ] ) for zfhash in zonefile_hashes ] ) zonefile_txids = dict ( [ ( zfhash , missing_zfinfo [ zfhash ] [ 'txid' ] ) for zfhash in zonefile_hashes ] ) zonefile_block_heights = dict ( [ ( zfhash , missing_zfinfo [ zfhash ] [ 'block_heights' ] ) for zfhash in zonefile_hashes ] ) zonefile_origins = self . find_zonefile_origins ( missing_zfinfo , peer_hostports ) for i in xrange ( 0 , len ( zonefile_hashes ) ) : zfhash = zonefile_hashes [ i ] present = is_zonefile_cached ( zfhash , self . zonefile_dir , validate = True ) if present : log . debug ( "%s: zonefile %s already cached.  Marking present" % ( self . hostport , zfhash ) ) zonefile_hashes [ i ] = None self . set_zonefile_present ( zfhash , min ( zonefile_block_heights [ zfhash ] ) , path = path ) zonefile_hashes = filter ( lambda zfh : zfh is not None , zonefile_hashes ) if len ( zonefile_hashes ) > 0 : log . debug ( "%s: missing %s unique zonefiles" % ( self . hostport , len ( zonefile_hashes ) ) ) while len ( zonefile_hashes ) > 0 and self . running : zfhash = zonefile_hashes [ 0 ] zfnames = zonefile_names [ zfhash ] zftxid = zonefile_txids [ zfhash ] peers = missing_zfinfo [ zfhash ] [ 'peers' ] peers = atlas_rank_peers_by_health ( peer_list = peers , with_zero_requests = True ) if len ( peers ) > 0 : log . debug ( "%s: zonefile %s available from %s peers (%s...)" % ( self . hostport , zfhash , len ( peers ) , "," . join ( peers [ : min ( 5 , len ( peers ) ) ] ) ) ) for peer_hostport in peers : if zfhash not in zonefile_origins [ peer_hostport ] : log . debug ( "%s not available from %s" % ( zfhash , peer_hostport ) ) continue peer_zonefile_hashes = [ ] for zfh in zonefile_origins [ peer_hostport ] : if zfh in zonefile_hashes : peer_zonefile_hashes . append ( zfh ) if len ( peer_zonefile_hashes ) == 0 : log . debug ( "%s: No zonefiles available from %s" % ( self . hostport , peer_hostport ) ) continue log . debug ( "%s: get %s zonefiles from %s" % ( self . hostport , len ( peer_zonefile_hashes ) , peer_hostport ) ) zonefiles = atlas_get_zonefiles ( self . hostport , peer_hostport , peer_zonefile_hashes , peer_table = peer_table ) if zonefiles is not None : stored_zfhashes = self . store_zonefiles ( zonefile_names , zonefiles , zonefile_txids , zonefile_block_heights , peer_zonefile_hashes , peer_hostport , path ) log . debug ( "Stored %s zonefiles" % len ( stored_zfhashes ) ) for zfh in stored_zfhashes : if zfh in peer_zonefile_hashes : peer_zonefile_hashes . remove ( zfh ) if zfh in zonefile_hashes : zonefile_hashes . remove ( zfh ) num_fetched += 1 else : log . debug ( "%s: no data received from %s" % ( self . hostport , peer_hostport ) ) with AtlasPeerTableLocked ( ) as ptbl : for zfh in peer_zonefile_hashes : log . debug ( "%s: %s did not have %s" % ( self . hostport , peer_hostport , zfh ) ) atlas_peer_set_zonefile_status ( peer_hostport , zfh , False , zonefile_bits = missing_zfinfo [ zfh ] [ 'indexes' ] , peer_table = ptbl ) if zfh in zonefile_origins [ peer_hostport ] : zonefile_origins [ peer_hostport ] . remove ( zfh ) if zfhash in zonefile_hashes : zonefile_hashes . remove ( zfhash ) if len ( zonefile_hashes ) > 0 or num_fetched > 0 : log . debug ( "%s: fetched %s zonefiles" % ( self . hostport , num_fetched ) ) return num_fetched
Run one step of this algorithm:
        * find the set of missing zonefiles
        * try to fetch each of them
        * store them
        * update our zonefile database

        Fetch rarest zonefiles first, but batch
        whenever possible.

        Return the number of zonefiles fetched