def filter_record(records):

    scheme = {
        'interaction': lambda r: r.interaction in ['call', 'text'],
        'direction': lambda r: r.direction in ['in', 'out'],
        'correspondent_id': lambda r: r.correspondent_id is not None,
        'datetime': lambda r: isinstance(r.datetime, datetime),
        'call_duration': lambda r: isinstance(r.call_duration, (int, float)) if r.interaction == 'call' else True,
    }

    ignored = OrderedDict([
        ('all', 0),
        ('interaction', 0),
        ('direction', 0),
        ('correspondent_id', 0),
        ('datetime', 0),
        ('call_duration', 0),
    ])

    bad_records = []

    def _filter(records):
        global removed
        for r in records:
            valid = True
            for key, test in scheme.iteritems():
                if not test(r):
                    ignored[key] += 1
                    bad_records.append(r)
                    valid = False  # Not breaking, we count all fields with errors

            if valid is True:
                yield r
            else:
                ignored['all'] += 1

    return list(_filter(records)), ignored, bad_records
Filter records and remove items with missing or inconsistent fields
dataset/ETHPy150Open yvesalexandre/bandicoot/bandicoot/io.py/filter_record