def remove_outliers(lodol, actually_keep=True):
    bad_indexes = set()
    bad_keys = set()
    DEVIATIONS = 3
    MAX_LENGTH = 10000
    
    for data in lodol:
        #print(data['name'])
        #print([e for e in data['data'] if isinstance(e, (str, unicode))])
        if isinstance(data['data'][0], (int, float)):
            print(data['name'])
            mean = statistics.mean(data['data'])
            std = statistics.stdev(data['data'])
            evils = 0
            for index, value in enumerate(data['data']):
                if mean - DEVIATIONS*std > value or value > mean + DEVIATIONS*std:
                    bad_keys.add(data['name'])
                    bad_indexes.add(index)
                    evils += 1
            #print(data['name'], mean-4*std, mean+4*std, evils)
                    
    total_indexes = len(lodol[0]['data'])
    reduced_indexes = total_indexes - len(bad_indexes)
    print("Bad indexes:", len(bad_indexes), "/", total_indexes)
    '''
    I have a list of numbers Z from 0 to N
    I have a list of J numbers (where J < N) randomly distributed throughout Z
    I wish to remove K numbers from Z, without drawing from any number in J
    '''
    if reduced_indexes > MAX_LENGTH:
        stride = total_indexes / MAX_LENGTH
        for an_index in xrange(0, total_indexes, stride):
            keep_index = random.randint(0, stride-1)
            for offset in xrange(0, stride):
                if keep_index != offset:
                    bad_indexes.add(min(total_indexes, an_index+offset))
    if actually_keep:
        bad_indexes = set()
        bad_keys = set()
    print("Trimmed indexes:", len(bad_indexes), "/", total_indexes)    
    #print("Contributing keys:", ', '.join(bad_keys))
    key_names = [row['name'] for row in lodol]
    short_key_names = shortest_unique_strings(key_names)
    key_name_map = dict(zip(key_names, short_key_names))
    for data in lodol:
        data['data'] = [v for i, v in enumerate(data['data']) if i not in bad_indexes]
        #inter = data['name'].split('.', 2)[2]
        #if '.' in inter:
        #    category, name = inter.rsplit('.', 1)
        #else:
        #    category, name = inter, inter
        #category = category.replace('.', ' ')
        #data['pretty'] = category.title() + ": "+name.title()
        data['pretty'] = key_name_map[data['name']] 
        print(data['pretty'])
        if isinstance(data['data'][0], (float, int)):
            data['type'] = 'number'
        else:
            data['type'] = 'text'
    lodol.sort(key= lambda e: e['pretty'])
    return key_name_map
def remove_outliers ( lodol , actually_keep = True ) : bad_indexes = set ( ) bad_keys = set ( ) DEVIATIONS = 3 MAX_LENGTH = 10000 for data in lodol : if isinstance ( data [ 'data' ] [ 0 ] , ( int , float ) ) : print ( data [ 'name' ] ) mean = statistics . mean ( data [ 'data' ] ) std = statistics . stdev ( data [ 'data' ] ) evils = 0 for index , value in enumerate ( data [ 'data' ] ) : if mean - DEVIATIONS * std > value or value > mean + DEVIATIONS * std : bad_keys . add ( data [ 'name' ] ) bad_indexes . add ( index ) evils += 1 total_indexes = len ( lodol [ 0 ] [ 'data' ] ) reduced_indexes = total_indexes - len ( bad_indexes ) print ( "Bad indexes:" , len ( bad_indexes ) , "/" , total_indexes ) if reduced_indexes > MAX_LENGTH : stride = total_indexes / MAX_LENGTH for an_index in xrange ( 0 , total_indexes , stride ) : keep_index = random . randint ( 0 , stride - 1 ) for offset in xrange ( 0 , stride ) : if keep_index != offset : bad_indexes . add ( min ( total_indexes , an_index + offset ) ) if actually_keep : bad_indexes = set ( ) bad_keys = set ( ) print ( "Trimmed indexes:" , len ( bad_indexes ) , "/" , total_indexes ) key_names = [ row [ 'name' ] for row in lodol ] short_key_names = shortest_unique_strings ( key_names ) key_name_map = dict ( zip ( key_names , short_key_names ) ) for data in lodol : data [ 'data' ] = [ v for i , v in enumerate ( data [ 'data' ] ) if i not in bad_indexes ] data [ 'pretty' ] = key_name_map [ data [ 'name' ] ] print ( data [ 'pretty' ] ) if isinstance ( data [ 'data' ] [ 0 ] , ( float , int ) ) : data [ 'type' ] = 'number' else : data [ 'type' ] = 'text' lodol . sort ( key = lambda e : e [ 'pretty' ] ) return key_name_map
I have a list of numbers Z from 0 to N
    I have a list of J numbers (where J < N) randomly distributed throughout Z
    I wish to remove K numbers from Z, without drawing from any number in J