Analyzes lines in CSV format.

    Expects the first line to be a header line starting with headers_start_with e.g. "ID,experimental value, prediction 1 value, prediction 2 value,"
    Record IDs are expected in the first column.
    Experimental values are expected in the second column.
    Predicted values are expected in the subsequent columns.

    :param csv_lines: Lines read from a CSV file containing experimental and predicted data for some dataset.
    :param expect_negative_correlation: Indicates wheter or not the predictions have the same or opposite sign as the experimental data.
    :param STDev_cutoff: Number of standard deviations as a threshold for significance cut-off. Used for both predictions and experiments.
    :param headers_start_with: A string used to distinguish the header line.
    :param comments_start_with: A string used to distinguish lines containing comments to be ignored.
    :param separator: Column separator. For CSV files, ',' should be used. For TSV files, '\t' should be used.

def parse_csv(csv_lines, expect_negative_correlation = False, STDev_cutoff = 1.0, headers_start_with = 'ID', comments_start_with = None, separator = ','):
    """
    Analyzes lines in CSV format.

    Expects the first line to be a header line starting with headers_start_with e.g. "ID,experimental value, prediction 1 value, prediction 2 value,"
    Record IDs are expected in the first column.
    Experimental values are expected in the second column.
    Predicted values are expected in the subsequent columns.

    :param csv_lines: Lines read from a CSV file containing experimental and predicted data for some dataset.
    :param expect_negative_correlation: Indicates wheter or not the predictions have the same or opposite sign as the experimental data.
    :param STDev_cutoff: Number of standard deviations as a threshold for significance cut-off. Used for both predictions and experiments.
    :param headers_start_with: A string used to distinguish the header line.
    :param comments_start_with: A string used to distinguish lines containing comments to be ignored.
    :param separator: Column separator. For CSV files, ',' should be used. For TSV files, '\t' should be used.
    """
    data = {}
    data['experimental'] = {}
    data['experimental']['list'] = []
    data['experimental']['dict'] = {}
    data['predictions'] = {}

    if comments_start_with != None:
        lines = [l for l in csv_lines if l.strip() and not(l.strip().startswith(comments_start_with))]
    else:
        lines = [l for l in csv_lines if l.strip()]

    headers = lines[0]
    assert(headers.startswith(headers_start_with))
    headers = headers.split(separator)
    num_fields = len(headers)
    prediction_names = headers[2:]

    skipped_cases = []
    for line in lines[1:]:

        fields = [t.strip() for t in line.split(separator)]
        assert(len(fields) == num_fields)

        mut_name = fields[0]
        expt_value = fields[1]

        # The script as is requires values for all records. We skip any which are missing some data.
        predicted_values = [t.strip() for t in fields[2:] if t.strip()]
        if(len(predicted_values) != len(fields) - 2):
            skipped_cases.append(fields[0])
            continue

        data['experimental']['dict'][mut_name] = float(expt_value)
        data['experimental']['list'].append(float(expt_value))
        predictions = fields[2:]
        counter = 0
        for value in predictions:
            counter += 1
            prediction_id = counter
            prediction_name = prediction_names[counter - 1]

            data['predictions'][prediction_id] = data['predictions'].get(prediction_id, dict(name = prediction_name, id = prediction_id))
            prediction_cases = data['predictions'][prediction_id]

            prediction_cases['list'] = prediction_cases.get('list', [])
            prediction_cases['dict'] = prediction_cases.get('dict', {})

            prediction_cases['dict']['mut_name'] = float(value)
            prediction_cases['list'].append(float(value))

    data['skipped_cases'] = skipped_cases
    for prediction_id, prediction_cases in data['predictions'].iteritems():
        prediction_cases['array'] = np.array(prediction_cases['list'])
        pos_STDev, neg_STDev = get_symmetrical_std_devs(prediction_cases['array'])
        prediction_cases['positive significance threshold'] = pos_STDev
        prediction_cases['negative significance threshold'] = neg_STDev

    data['experimental']['array'] = np.array(data['experimental']['list'])
    pos_STDev, neg_STDev = get_symmetrical_std_devs(data['experimental']['array'])
    data['experimental']['positive significance threshold'] = pos_STDev
    data['experimental']['negative significance threshold'] = neg_STDev

    for prediction_id, prediction_cases in sorted(data['predictions'].iteritems()):
        slope_through_origin, r_value_through_origin = least_squares_fixed_origin(prediction_cases['array'], data['experimental']['array'])
        slope, intercept, r_value, p_value, std_err = stats.linregress(prediction_cases['array'], data['experimental']['array'])

        prediction_cases.update(dict(
            prediction_id = prediction_id,
            slope_through_origin = slope_through_origin,
            R_value_through_origin = r_value_through_origin,
            slope = slope,
            R_value = r_value,
        ))

        warnings = []
        mae, scaled_mae, warning = calculate_mae(prediction_cases, data['experimental'])
        accuracy, specificity, sensitivity, significance_specificity, significance_sensitivity, significant_beneficient_sensitivity, significant_beneficient_specificity, significant_beneficient_sensitivity_n, significant_beneficient_specificity_n = determine_correct_sign(prediction_cases, data['experimental'], expect_negative_correlation, STDev_cutoff)
        if warning:
            warnings = [warning]

        prediction_cases.update(dict(
            STDev_cutoff = STDev_cutoff,
            warnings = warnings,
            MAE = mae,
            scaled_MAE = scaled_mae,
            accuracy = accuracy,
            specificity = specificity,
            sensitivity = sensitivity,
            significance_specificity = significance_specificity,
            significance_sensitivity = significance_sensitivity,
            significant_beneficient_sensitivity = (significant_beneficient_sensitivity, significant_beneficient_sensitivity_n),
            significant_beneficient_specificity = (significant_beneficient_specificity, significant_beneficient_specificity_n),
        ))
    return data