def iterdupes(self, compare=None, filt=None):
        ''' streaming item iterator with low overhead duplicate file detection

            Parameters:
                - compare       compare function between files (defaults to md5sum)
        '''
        if not compare: compare = self.md5sum
        seen_siz = {}       ## store size   -> first seen filename
        seen_sum = {}       ## store chksum -> first seen filename
        size_func = lambda x: os.stat(x).st_size
        for (fsize, f) in self.iteritems(want_dirs=False, func=size_func, filt=filt):
            if fsize not in seen_siz:    ## state 1: no previous size collisions
                seen_siz[fsize] = f
                continue
            else:
                if seen_siz[fsize]:      ## state 2: defined key => str (initial, unscanned path)
                    chksum = compare(seen_siz[fsize])
                    if chksum in seen_sum:  yield (chksum, seen_siz[fsize])
                    else:                   seen_sum[chksum] = seen_siz[fsize]
                    seen_siz[fsize] = None

                ## state 3: defined key => None (already scanned path, no-op)
                chksum = compare(f)
                if chksum in seen_sum:
                    ## if it's a dupe, check if the first one was ever yielded then yield
                    if seen_sum[chksum]:
                        yield (chksum, seen_sum[chksum])
                        seen_sum[chksum] = None
                    yield (chksum, f)
                else:
                    ## if not, set the initial filename
                    seen_sum[chksum] = f
def iterdupes ( self , compare = None , filt = None ) : if not compare : compare = self . md5sum seen_siz = { } seen_sum = { } size_func = lambda x : os . stat ( x ) . st_size for ( fsize , f ) in self . iteritems ( want_dirs = False , func = size_func , filt = filt ) : if fsize not in seen_siz : seen_siz [ fsize ] = f continue else : if seen_siz [ fsize ] : chksum = compare ( seen_siz [ fsize ] ) if chksum in seen_sum : yield ( chksum , seen_siz [ fsize ] ) else : seen_sum [ chksum ] = seen_siz [ fsize ] seen_siz [ fsize ] = None chksum = compare ( f ) if chksum in seen_sum : if seen_sum [ chksum ] : yield ( chksum , seen_sum [ chksum ] ) seen_sum [ chksum ] = None yield ( chksum , f ) else : seen_sum [ chksum ] = f
streaming item iterator with low overhead duplicate file detection

            Parameters:
                - compare       compare function between files (defaults to md5sum)