import sys
import os
import glob
import string
import logging
import yaml
import yamlordereddictloader

def extract_data(c, param_map):
    """
    Method to generate a CPAC input subject list
    python file. The method extracts anatomical
    and functional data for each site( if multiple site)
    and/or scan parameters for each site and put it into
    a data structure read by python

    Example:
    subjects_list =[
       {
        'subject_id' : '0050386',
        'unique_id' : 'session_1',
        'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',
        'rest':{
            'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',
            'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',
            }
        'scan_parameters':{
            'tr': '2',
            'acquisition': 'alt+z2',
            'reference': '17',
            'first_tr': '',
            'last_tr': '',
            }
        },
    ]

    or

    subjects_list =[
       {
        'subject_id' : '0050386',
        'unique_id' : 'session_1',
        'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',
        'rest':{
            'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',
            'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',
            }
          },
    ]

    """

    #method to read each line of the file into list
    #returns list
    def get_list(arg):
        if isinstance(arg, list):
            ret_list = arg
        else:
            ret_list = [fline.rstrip('\r\n') for fline in open(arg, 'r').readlines()]

        return ret_list

    exclusion_list = []
    if c.exclusionSubjectList is not None:
        exclusion_list = get_list(c.exclusionSubjectList)

    subject_list = []
    if c.subjectList is not None:
        subject_list = get_list(c.subjectList)

    #check if Template is correct
    def checkTemplate(template):

        if template.count('%s') != 2:
            msg = "Please provide '%s' in the template" \
                  "where your site and subjects are present"\
                  "Please see examples"
            logging.exception(msg)
            raise Exception(msg)

        filename, ext = os.path.splitext(os.path.basename(template))
        ext = os.path.splitext(filename)[1] + ext

        if ext not in [".nii", ".nii.gz"]:
            msg = "Invalid file name", os.path.basename(template)
            logging.exception(msg)
            raise Exception(msg)

    def get_site_list(path):
        base, relative = path.split('%s')
        sites = os.listdir(base)
        return sites

    def check_length(scan_name, file_name):

        if len(file_name) > 30:
            msg = "filename- %s is too long."\
                   "It should not be more than 30 characters."%(file_name)
            logging.exception(msg)
            raise Exception(msg)

        if len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0])>= 40:
            msg = "scan name %s is too long."\
                  "It should not be more than 20 characters"\
                  %(scan_name.replace("_"+os.path.splitext(os.path.splitext(file_name)[0])[0], ''))
            logging.exception(msg)
            raise Exception(msg)

    def create_site_subject_mapping(base, relative):

        #mapping between site and subject
        site_subject_map = {}
        base_path_list = []

        if c.siteList is not None:
            site_list = get_list(c.siteList)
        else:
            site_list = get_site_list(base)

        for site in site_list:
            paths = glob.glob(string.replace(base, '%s', site))
            base_path_list.extend(paths)
            for path in paths:
                for sub in os.listdir(path):
                    #check if subject is present in subject_list
                    if subject_list:
                        if sub in subject_list and sub not in exclusion_list:
                            site_subject_map[sub] = site
                    elif sub not in exclusion_list:
                        if sub not in '.DS_Store':
                            site_subject_map[sub] = site

        return base_path_list, site_subject_map

    #method to split the input template path
    #into base, path before subject directory
    #and relative, path after subject directory
    def getPath(template):

        checkTemplate(template)
        base, relative = template.rsplit("%s", 1)
        base, subject_map = create_site_subject_mapping(base, relative)
        base.sort()
        relative = relative.lstrip("/")
        return base, relative, subject_map

    #get anatomical base path and anatomical relative path
    anat_base, anat_relative = getPath(c.anatomicalTemplate)[:2]

    #get functional base path, functional relative path and site-subject map
    func_base, func_relative, subject_map = getPath(c.functionalTemplate)

    if not anat_base:
        msg = "Anatomical Data template incorrect. No such file or directory %s", anat_base
        logging.exception(msg)
        raise Exception(msg)

    if not func_base:
        msg = "Functional Data template incorrect. No such file or directory %s, func_base"
        logging.exception(msg)
        raise Exception(msg)

    if len(anat_base) != len(func_base):
        msg1 = "Some sites are missing, Please check your template"\
              , anat_base, "!=", func_base
        logging.exception(msg1)

        msg2 = " Base length Unequal. Some sites are missing."\
               "extract_data doesn't script support this.Please" \
               "Provide your own subjects_list file"
        logging.exception(msg2)
        raise Exception(msg2)

    #calculate the length of relative paths(path after subject directory)
    func_relative_len = len(func_relative.split('/'))
    anat_relative_len = len(anat_relative.split('/'))

    def check_for_sessions(relative_path, path_length):
        """
        Method to check if there are sessions present

        """
        #default
        session_present = False
        session_path = 'session_1'

        #session present if path_length is equal to 3
        if path_length == 3:
            relative_path_list = relative_path.split('/')
            session_path = relative_path_list[0]
            relative_path = string.join(relative_path_list[1:], "/")
            session_present = True
        elif path_length > 3:
            msg = "extract_data script currently doesn't support this directory structure."\
                  "Please provide the subjects_list file to run CPAC."\
                  "For more information refer to manual"
            logging.exception(msg)
            raise Exception(msg)
        return session_present, session_path, relative_path

    func_session_present, func_session_path, func_relative = \
        check_for_sessions(func_relative, func_relative_len)

    anat_session_present, anat_session_path, anat_relative = \
        check_for_sessions(anat_relative, anat_relative_len)

    f = open(os.path.join(c.outputSubjectListLocation, "CPAC_subject_list_%s.yml" % c.subjectListName), 'wb')

    def fetch_path(i, anat_sub, func_sub, session_id):
        """
        Method to extract anatomical and functional
        path for a session and print to file

        Parameters
        ----------
        i : int
            index of site
        anat_sub : string
            string containing subject/ concatenated
            subject-session path for anatomical file
        func_sub: string
            string containing subject/ concatenated
            subject-session path for functional file
        session_id: string
            session

        Raises
        ------
        Exception
        """

        try:

            def print_begin_of_file(sub, session_id):
                print("-", file=f)
                print("    subject_id: '" + sub + "'", file=f)
                print("    unique_id: '" + session_id + "'", file=f)

            def print_end_of_file(sub):
                if param_map is not None:
                    try:
                        logging.debug("site for sub %s -> %s" %(sub, subject_map.get(sub)))
                        logging.debug("scan parameters for the above site %s"%param_map.get(subject_map.get(sub)))
                        print("    scan_parameters:", file=f)
                        print("        tr: '" + param_map.get(subject_map.get(sub))[4] + "'", file=f)
                        print("        acquisition: '" + param_map.get(subject_map.get(sub))[0] + "'", file=f)
                        print("        reference: '" + param_map.get(subject_map.get(sub))[3] + "'", file=f)
                        print("        first_tr: '" + param_map.get(subject_map.get(sub))[1] + "'", file=f)
                        print("        last_tr: '" + param_map.get(subject_map.get(sub))[2] + "'", file=f)
                    except:
                        msg = " No Parameter values for the %s site is defined in the scan"\
                              " parameters csv file" %subject_map.get(sub)
                        raise ValueError(msg)

            #get anatomical file
            anat_base_path = os.path.join(anat_base[i], anat_sub)
            func_base_path = os.path.join(func_base[i], func_sub)

            anat = None
            func = None

            anat = glob.glob(os.path.join(anat_base_path, anat_relative))
            func = glob.glob(os.path.join(func_base_path, func_relative))

            if anat and func:
                print_begin_of_file(anat_sub.split("/")[0], session_id)
                print("    anat: '" + os.path.realpath(anat[0]) + "'", file=f)
                print("    rest: ", file=f)

                #iterate for each rest session
                for iter in func:
                    #get scan_id
                    iterable = os.path.splitext(os.path.splitext(iter.replace(func_base_path, '').lstrip("/"))[0])[0]
                    iterable = iterable.replace("/", "_")
                    check_length(iterable, os.path.basename(os.path.realpath(iter)))
                    print("      " + iterable + ": '" + os.path.realpath(iter) + "'", file=f)

                print_end_of_file(anat_sub.split("/")[0])

            else:
                logging.debug("skipping subject %s"%anat_sub.split("/")[0])

        except ValueError:

            logging.exception(ValueError.message)
            raise

        except Exception as e:

            err_msg = 'Exception while felching anatomical and functional ' \
                      'paths: \n' + str(e)

            logging.exception(err_msg)
            raise Exception(err_msg)

    def walk(index, sub):
        """
        Method which walks across each subject
        path in the data site path

        Parameters
        ----------
        index : int
            index of site
        sub : string
            subject_id

        Raises
        ------
        Exception
        """
        try:

            if func_session_present:
                #if there are sessions
                if "*" in func_session_path:
                    session_list = glob.glob(os.path.join(func_base[index], os.path.join(sub, func_session_path)))
                else:
                    session_list = [func_session_path]

                if session_list:
                    for session in session_list:
                        session_id = os.path.basename(session)
                        if anat_session_present:
                            if func_session_path == anat_session_path:
                                fetch_path(index, os.path.join(sub, session_id), os.path.join(sub, session_id), session_id)
                            else:
                                fetch_path(index, os.path.join(sub, anat_session_path), os.path.join(sub, session_id), session_id)
                        else:
                            fetch_path(index, sub, os.path.join(sub, session_id), session_id)
                else:
                    logging.debug("Skipping subject %s", sub)

            else:
                logging.debug("No sessions")
                session_id = ''
                fetch_path(index, sub, sub, session_id)

        except Exception:

            logging.exception(Exception.message)
            raise

        except:

            err_msg = 'Please make sessions are consistent across all ' \
                      'subjects.\n\n'

            logging.exception(err_msg)
            raise Exception(err_msg)


    try:
        for i in range(len(anat_base)):
            for sub in os.listdir(anat_base[i]):
                #check if subject is present in subject_list
                if subject_list:
                    if sub in subject_list and sub not in exclusion_list:
                        logging.debug("extracting data for subject: %s", sub)
                        walk(i, sub)
                #check that subject is not in exclusion list
                elif sub not in exclusion_list and sub not in '.DS_Store':
                    logging.debug("extracting data for subject: %s", sub)
                    walk(i, sub)


        name = os.path.join(c.outputSubjectListLocation, 'CPAC_subject_list.yml')
        print("Extraction Successfully Completed...Input Subjects_list for CPAC - %s" % name)

    except Exception:

        logging.exception(Exception.message)
        raise

    finally:
        f.close()


def generate_supplementary_files(data_config_outdir, data_config_name):

    """
    Method to generate phenotypic template file
    and subject list for group analysis
    """

    import os
    from sets import Set
    import csv

    data_config_path = os.path.join(data_config_outdir, data_config_name)

    try:
        subjects_list = yaml.safe_load(open(data_config_path, 'r'))
    except:
        err = "\n\n[!] Data configuration file couldn't be read!\nFile " \
              "path: {0}\n".format(data_config_path)

    subject_scan_set = Set()
    subID_set = Set()
    session_set = Set()
    subject_set = Set()
    scan_set = Set()
    data_list = []

    try:
        for sub in subjects_list:
            if sub['unique_id']:
                subject_id = sub['subject_id'] + "_" + sub['unique_id']
            else:
                subject_id = sub['subject_id']

            try:
                for scan in sub['func']:
                    subject_scan_set.add((subject_id, scan))
                    subID_set.add(sub['subject_id'])
                    session_set.add(sub['unique_id'])
                    subject_set.add(subject_id)
                    scan_set.add(scan)
            except KeyError:
                try:
                    for scan in sub['rest']:
                        subject_scan_set.add((subject_id, scan))
                        subID_set.add(sub['subject_id'])
                        session_set.add(sub['unique_id'])
                        subject_set.add(subject_id)
                        scan_set.add(scan)
                except KeyError:
                    # one of the participants in the subject list has no
                    # functional scans
                    subID_set.add(sub['subject_id'])
                    session_set.add(sub['unique_id'])
                    subject_set.add(subject_id)

    except TypeError as e:
        print('Subject list could not be populated!')
        print('This is most likely due to a mis-formatting in your '\
              'inclusion and/or exclusion subjects txt file or your '\
              'anatomical and/or functional path templates.')
        print('Error: %s' % e)
        err_str = 'Check formatting of your anatomical/functional path '\
                  'templates and inclusion/exclusion subjects text files'
        raise TypeError(err_str)

    for item in subject_scan_set:
        list1 = []
        list1.append(item[0] + "/" + item[1])
        for val in subject_set:
            if val in item:
                list1.append(1)
            else:
                list1.append(0)

        for val in scan_set:
            if val in item:
                list1.append(1)
            else:
                list1.append(0)

        data_list.append(list1)

    # generate the phenotypic file templates for group analysis
    file_name = os.path.join(data_config_outdir, 'phenotypic_template_%s.csv'
                             % data_config_name)

    try:
        f = open(file_name, 'wb')
    except:
        print('\n\nCPAC says: I couldn\'t save this file to your drive:\n')
        print(file_name, '\n\n')
        print('Make sure you have write access? Then come back. Don\'t ' \
                'worry.. I\'ll wait.\n\n')
        raise IOError

    writer = csv.writer(f)

    writer.writerow(['participant', 'EV1', '..'])
    for sub in sorted(subID_set):
        writer.writerow([sub, ''])

    f.close()

    print("Template Phenotypic file for group analysis - %s" % file_name)

    """
    # generate the phenotypic file templates for repeated measures
    if (len(session_set) > 1) and (len(scan_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_mult_sessions_and_scans_%s.csv' \
                % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                    'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)
        writer.writerow(['participant', 'session', 'series', 'EV1', '..'])

        for session in sorted(session_set):
            for scan in sorted(scan_set):
                for sub in sorted(subID_set):
                    writer.writerow([sub, session, scan, ''])

        f.close()

        print "Template Phenotypic file for group analysis with repeated " \
              "measures (multiple sessions and scans) - %s" % file_name

    if (len(session_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_multiple_sessions_%s.csv' % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                  'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)

        writer.writerow(['participant', 'session', 'EV1', '..'])

        for session in sorted(session_set):
            for sub in sorted(subID_set):
                writer.writerow([sub, session, ''])

        f.close()

        print "Template Phenotypic file for group analysis with repeated " \
              "measures (multiple sessions) - %s" % file_name

    if (len(scan_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_multiple_scans_%s.csv' % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                  'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)

        writer.writerow(['participant', 'series', 'EV1', '..'])

        for scan in sorted(scan_set):
            for sub in sorted(subID_set):
                writer.writerow([sub, scan, ''])

        f.close()

    print "Template Phenotypic file for group analysis with repeated " \
          "measures (multiple scans) - %s" % file_name
    """

    # generate the group analysis subject lists
    file_name = os.path.join(data_config_outdir,
                             'participant_list_group_analysis_%s.txt'
                             % data_config_name)

    try:
        with open(file_name, 'w') as f:
            for sub in sorted(subID_set):
                print(sub, file=f)
    except:
        print('\n\nCPAC says: I couldn\'t save this file to your drive:\n')
        print(file_name, '\n\n')
        print('Make sure you have write access? Then come back. Don\'t ' \
              'worry.. I\'ll wait.\n\n')
        raise IOError

    print("Participant list required later for group analysis - %s\n\n" \
          % file_name)


def read_csv(csv_input):
    """
    Method to read csv file
     'Acquisition'
     'Reference'
     'Site'
     'TR (seconds)'

    """
    import csv
    from collections import defaultdict
    try:
        reader = csv.DictReader(open(csv_input, "U"))

        dict_labels = defaultdict(list)
        for line in reader:
            csv_dict = dict((k.lower(), v) for k, v in line.items())
            dict_labels[csv_dict.get('site')] = [
                csv_dict[key] for key in sorted(list(
                    csv_dict.keys()
                )) if key != 'site' and key != 'scan'
            ]

        if len(dict_labels) < 1:
            msg ="Scan Parameters File is either empty"\
                 "or missing header"
            logging.exception(msg)
            raise Exception(msg)

        return dict_labels

    except IOError:
        msg = "Error reading the csv file %s", csv_input
        logging.exception(msg)
        raise Exception(msg)
    except:
        msg = "Error reading scan parameters csv. Make sure you are using the correct template"
        logging.exception(msg)
        raise Exception(msg)


"""
Class to set dictionary keys as map attributes
"""
class Configuration(object):
    def __init__(self, config_map):
        for key in config_map:
            if config_map[key] == 'None':
                config_map[key] = None
            setattr(self, key, config_map[key])


def run(data_config):
    """
    Run method takes data_config
    file as the input argument
    """
    root = logging.getLogger()
    if root.handlers:
        for handler in root.handlers:
            root.removeHandler(handler)
    logging.basicConfig(filename=os.path.join(os.getcwd(), 'extract_data_logs.log'), filemode='w', level=logging.DEBUG,\
                    format="%(levelname)s %(asctime)s %(lineno)d %(message)s")

    print("For any errors or messages check the log file - %s"\
           % os.path.join(os.getcwd(), 'extract_data_logs.log'))

    c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), 'r')))

    if c.scanParametersCSV is not None:
        s_param_map = read_csv(c.scanParametersCSV)
    else:
        logging.debug("no scan parameters csv included\n"\
              "make sure you turn off slice timing correction option\n"\
              "in CPAC configuration\n")
        s_param_map = None

    generate_supplementary_files(c.outputSubjectListLocation, c.subjectListName)


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python extract_data.py data_config.yml")
        sys.exit()
    else:
        run(sys.argv[1])

import sys
import os
import glob
import string
import logging
import yaml
import yamlordereddictloader

def extract_data(c, param_map):
    """
    Method to generate a CPAC input subject list
    python file. The method extracts anatomical
    and functional data for each site( if multiple site)
    and/or scan parameters for each site and put it into
    a data structure read by python

    Example:
    subjects_list =[
       {
        'subject_id' : '0050386',
        'unique_id' : 'session_1',
        'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',
        'rest':{
            'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',
            'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',
            }
        'scan_parameters':{
            'tr': '2',
            'acquisition': 'alt+z2',
            'reference': '17',
            'first_tr': '',
            'last_tr': '',
            }
        },
    ]

    or

    subjects_list =[
       {
        'subject_id' : '0050386',
        'unique_id' : 'session_1',
        'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',
        'rest':{
            'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',
            'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',
            }
          },
    ]

    """

    #method to read each line of the file into list
    #returns list
    def get_list(arg):
        if isinstance(arg, list):
            ret_list = arg
        else:
            ret_list = [fline.rstrip('\r\n') for fline in open(arg, 'r').readlines()]

        return ret_list

    exclusion_list = []
    if c.exclusionSubjectList is not None:
        exclusion_list = get_list(c.exclusionSubjectList)

    subject_list = []
    if c.subjectList is not None:
        subject_list = get_list(c.subjectList)

    #check if Template is correct
    def checkTemplate(template):

        if template.count('%s') != 2:
            msg = "Please provide '%s' in the template" \
                  "where your site and subjects are present"\
                  "Please see examples"
            logging.exception(msg)
            raise Exception(msg)

        filename, ext = os.path.splitext(os.path.basename(template))
        ext = os.path.splitext(filename)[1] + ext

        if ext not in [".nii", ".nii.gz"]:
            msg = "Invalid file name", os.path.basename(template)
            logging.exception(msg)
            raise Exception(msg)

    def get_site_list(path):
        base, relative = path.split('%s')
        sites = os.listdir(base)
        return sites

    def check_length(scan_name, file_name):

        if len(file_name) > 30:
            msg = "filename- %s is too long."\
                   "It should not be more than 30 characters."%(file_name)
            logging.exception(msg)
            raise Exception(msg)

        if len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0])>= 40:
            msg = "scan name %s is too long."\
                  "It should not be more than 20 characters"\
                  %(scan_name.replace("_"+os.path.splitext(os.path.splitext(file_name)[0])[0], ''))
            logging.exception(msg)
            raise Exception(msg)

    def create_site_subject_mapping(base, relative):

        #mapping between site and subject
        site_subject_map = {}
        base_path_list = []

        if c.siteList is not None:
            site_list = get_list(c.siteList)
        else:
            site_list = get_site_list(base)

        for site in site_list:
            paths = glob.glob(string.replace(base, '%s', site))
            base_path_list.extend(paths)
            for path in paths:
                for sub in os.listdir(path):
                    #check if subject is present in subject_list
                    if subject_list:
                        if sub in subject_list and sub not in exclusion_list:
                            site_subject_map[sub] = site
                    elif sub not in exclusion_list:
                        if sub not in '.DS_Store':
                            site_subject_map[sub] = site

        return base_path_list, site_subject_map

    #method to split the input template path
    #into base, path before subject directory
    #and relative, path after subject directory
    def getPath(template):

        checkTemplate(template)
        base, relative = template.rsplit("%s", 1)
        base, subject_map = create_site_subject_mapping(base, relative)
        base.sort()
        relative = relative.lstrip("/")
        return base, relative, subject_map

    #get anatomical base path and anatomical relative path
    anat_base, anat_relative = getPath(c.anatomicalTemplate)[:2]

    #get functional base path, functional relative path and site-subject map
    func_base, func_relative, subject_map = getPath(c.functionalTemplate)

    if not anat_base:
        msg = "Anatomical Data template incorrect. No such file or directory %s", anat_base
        logging.exception(msg)
        raise Exception(msg)

    if not func_base:
        msg = "Functional Data template incorrect. No such file or directory %s, func_base"
        logging.exception(msg)
        raise Exception(msg)

    if len(anat_base) != len(func_base):
        msg1 = "Some sites are missing, Please check your template"\
              , anat_base, "!=", func_base
        logging.exception(msg1)

        msg2 = " Base length Unequal. Some sites are missing."\
               "extract_data doesn't script support this.Please" \
               "Provide your own subjects_list file"
        logging.exception(msg2)
        raise Exception(msg2)

    #calculate the length of relative paths(path after subject directory)
    func_relative_len = len(func_relative.split('/'))
    anat_relative_len = len(anat_relative.split('/'))

    def check_for_sessions(relative_path, path_length):
        """
        Method to check if there are sessions present

        """
        #default
        session_present = False
        session_path = 'session_1'

        #session present if path_length is equal to 3
        if path_length == 3:
            relative_path_list = relative_path.split('/')
            session_path = relative_path_list[0]
            relative_path = string.join(relative_path_list[1:], "/")
            session_present = True
        elif path_length > 3:
            msg = "extract_data script currently doesn't support this directory structure."\
                  "Please provide the subjects_list file to run CPAC."\
                  "For more information refer to manual"
            logging.exception(msg)
            raise Exception(msg)
        return session_present, session_path, relative_path

    func_session_present, func_session_path, func_relative = \
        check_for_sessions(func_relative, func_relative_len)

    anat_session_present, anat_session_path, anat_relative = \
        check_for_sessions(anat_relative, anat_relative_len)

    f = open(os.path.join(c.outputSubjectListLocation, "CPAC_subject_list_%s.yml" % c.subjectListName), 'wb')

    def fetch_path(i, anat_sub, func_sub, session_id):
        """
        Method to extract anatomical and functional
        path for a session and print to file

        Parameters
        ----------
        i : int
            index of site
        anat_sub : string
            string containing subject/ concatenated
            subject-session path for anatomical file
        func_sub: string
            string containing subject/ concatenated
            subject-session path for functional file
        session_id: string
            session

        Raises
        ------
        Exception
        """

        try:

            def print_begin_of_file(sub, session_id):
                print("-", file=f)
                print("    subject_id: '" + sub + "'", file=f)
                print("    unique_id: '" + session_id + "'", file=f)

            def print_end_of_file(sub):
                if param_map is not None:
                    try:
                        logging.debug("site for sub %s -> %s" %(sub, subject_map.get(sub)))
                        logging.debug("scan parameters for the above site %s"%param_map.get(subject_map.get(sub)))
                        print("    scan_parameters:", file=f)
                        print("        tr: '" + param_map.get(subject_map.get(sub))[4] + "'", file=f)
                        print("        acquisition: '" + param_map.get(subject_map.get(sub))[0] + "'", file=f)
                        print("        reference: '" + param_map.get(subject_map.get(sub))[3] + "'", file=f)
                        print("        first_tr: '" + param_map.get(subject_map.get(sub))[1] + "'", file=f)
                        print("        last_tr: '" + param_map.get(subject_map.get(sub))[2] + "'", file=f)
                    except:
                        msg = " No Parameter values for the %s site is defined in the scan"\
                              " parameters csv file" %subject_map.get(sub)
                        raise ValueError(msg)

            #get anatomical file
            anat_base_path = os.path.join(anat_base[i], anat_sub)
            func_base_path = os.path.join(func_base[i], func_sub)

            anat = None
            func = None

            anat = glob.glob(os.path.join(anat_base_path, anat_relative))
            func = glob.glob(os.path.join(func_base_path, func_relative))

            if anat and func:
                print_begin_of_file(anat_sub.split("/")[0], session_id)
                print("    anat: '" + os.path.realpath(anat[0]) + "'", file=f)
                print("    rest: ", file=f)

                #iterate for each rest session
                for iter in func:
                    #get scan_id
                    iterable = os.path.splitext(os.path.splitext(iter.replace(func_base_path, '').lstrip("/"))[0])[0]
                    iterable = iterable.replace("/", "_")
                    check_length(iterable, os.path.basename(os.path.realpath(iter)))
                    print("      " + iterable + ": '" + os.path.realpath(iter) + "'", file=f)

                print_end_of_file(anat_sub.split("/")[0])

            else:
                logging.debug("skipping subject %s"%anat_sub.split("/")[0])

        except ValueError:

            logging.exception(ValueError.message)
            raise

        except Exception as e:

            err_msg = 'Exception while felching anatomical and functional ' \
                      'paths: \n' + str(e)

            logging.exception(err_msg)
            raise Exception(err_msg)

    def walk(index, sub):
        """
        Method which walks across each subject
        path in the data site path

        Parameters
        ----------
        index : int
            index of site
        sub : string
            subject_id

        Raises
        ------
        Exception
        """
        try:

            if func_session_present:
                #if there are sessions
                if "*" in func_session_path:
                    session_list = glob.glob(os.path.join(func_base[index], os.path.join(sub, func_session_path)))
                else:
                    session_list = [func_session_path]

                if session_list:
                    for session in session_list:
                        session_id = os.path.basename(session)
                        if anat_session_present:
                            if func_session_path == anat_session_path:
                                fetch_path(index, os.path.join(sub, session_id), os.path.join(sub, session_id), session_id)
                            else:
                                fetch_path(index, os.path.join(sub, anat_session_path), os.path.join(sub, session_id), session_id)
                        else:
                            fetch_path(index, sub, os.path.join(sub, session_id), session_id)
                else:
                    logging.debug("Skipping subject %s", sub)

            else:
                logging.debug("No sessions")
                session_id = ''
                fetch_path(index, sub, sub, session_id)

        except Exception:

            logging.exception(Exception.message)
            raise

        except:

            err_msg = 'Please make sessions are consistent across all ' \
                      'subjects.\n\n'

            logging.exception(err_msg)
            raise Exception(err_msg)


    try:
        for i in range(len(anat_base)):
            for sub in os.listdir(anat_base[i]):
                #check if subject is present in subject_list
                if subject_list:
                    if sub in subject_list and sub not in exclusion_list:
                        logging.debug("extracting data for subject: %s", sub)
                        walk(i, sub)
                #check that subject is not in exclusion list
                elif sub not in exclusion_list and sub not in '.DS_Store':
                    logging.debug("extracting data for subject: %s", sub)
                    walk(i, sub)


        name = os.path.join(c.outputSubjectListLocation, 'CPAC_subject_list.yml')
        print("Extraction Successfully Completed...Input Subjects_list for CPAC - %s" % name)

    except Exception:

        logging.exception(Exception.message)
        raise

    finally:
        f.close()


def generate_supplementary_files(data_config_outdir, data_config_name):

    """
    Method to generate phenotypic template file
    and subject list for group analysis
    """

    import os
    from sets import Set
    import csv

    data_config_path = os.path.join(data_config_outdir, data_config_name)

    try:
        subjects_list = yaml.safe_load(open(data_config_path, 'r'))
    except:
        err = "\n\n[!] Data configuration file couldn't be read!\nFile " \
              "path: {0}\n".format(data_config_path)

    subject_scan_set = Set()
    subID_set = Set()
    session_set = Set()
    subject_set = Set()
    scan_set = Set()
    data_list = []

    try:
        for sub in subjects_list:
            if sub['unique_id']:
                subject_id = sub['subject_id'] + "_" + sub['unique_id']
            else:
                subject_id = sub['subject_id']

            try:
                for scan in sub['func']:
                    subject_scan_set.add((subject_id, scan))
                    subID_set.add(sub['subject_id'])
                    session_set.add(sub['unique_id'])
                    subject_set.add(subject_id)
                    scan_set.add(scan)
            except KeyError:
                try:
                    for scan in sub['rest']:
                        subject_scan_set.add((subject_id, scan))
                        subID_set.add(sub['subject_id'])
                        session_set.add(sub['unique_id'])
                        subject_set.add(subject_id)
                        scan_set.add(scan)
                except KeyError:
                    # one of the participants in the subject list has no
                    # functional scans
                    subID_set.add(sub['subject_id'])
                    session_set.add(sub['unique_id'])
                    subject_set.add(subject_id)

    except TypeError as e:
        print('Subject list could not be populated!')
        print('This is most likely due to a mis-formatting in your '\
              'inclusion and/or exclusion subjects txt file or your '\
              'anatomical and/or functional path templates.')
        print('Error: %s' % e)
        err_str = 'Check formatting of your anatomical/functional path '\
                  'templates and inclusion/exclusion subjects text files'
        raise TypeError(err_str)

    for item in subject_scan_set:
        list1 = []
        list1.append(item[0] + "/" + item[1])
        for val in subject_set:
            if val in item:
                list1.append(1)
            else:
                list1.append(0)

        for val in scan_set:
            if val in item:
                list1.append(1)
            else:
                list1.append(0)

        data_list.append(list1)

    # generate the phenotypic file templates for group analysis
    file_name = os.path.join(data_config_outdir, 'phenotypic_template_%s.csv'
                             % data_config_name)

    try:
        f = open(file_name, 'wb')
    except:
        print('\n\nCPAC says: I couldn\'t save this file to your drive:\n')
        print(file_name, '\n\n')
        print('Make sure you have write access? Then come back. Don\'t ' \
                'worry.. I\'ll wait.\n\n')
        raise IOError

    writer = csv.writer(f)

    writer.writerow(['participant', 'EV1', '..'])
    for sub in sorted(subID_set):
        writer.writerow([sub, ''])

    f.close()

    print("Template Phenotypic file for group analysis - %s" % file_name)

    """
    # generate the phenotypic file templates for repeated measures
    if (len(session_set) > 1) and (len(scan_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_mult_sessions_and_scans_%s.csv' \
                % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                    'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)
        writer.writerow(['participant', 'session', 'series', 'EV1', '..'])

        for session in sorted(session_set):
            for scan in sorted(scan_set):
                for sub in sorted(subID_set):
                    writer.writerow([sub, session, scan, ''])

        f.close()

        print "Template Phenotypic file for group analysis with repeated " \
              "measures (multiple sessions and scans) - %s" % file_name

    if (len(session_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_multiple_sessions_%s.csv' % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                  'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)

        writer.writerow(['participant', 'session', 'EV1', '..'])

        for session in sorted(session_set):
            for sub in sorted(subID_set):
                writer.writerow([sub, session, ''])

        f.close()

        print "Template Phenotypic file for group analysis with repeated " \
              "measures (multiple sessions) - %s" % file_name

    if (len(scan_set) > 1):

        file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \
                '_measures_multiple_scans_%s.csv' % data_config_name)

        try:
            f = open(file_name, 'wb')
        except:
            print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'
            print file_name, '\n\n'
            print 'Make sure you have write access? Then come back. Don\'t ' \
                  'worry.. I\'ll wait.\n\n'
            raise IOError

        writer = csv.writer(f)

        writer.writerow(['participant', 'series', 'EV1', '..'])

        for scan in sorted(scan_set):
            for sub in sorted(subID_set):
                writer.writerow([sub, scan, ''])

        f.close()

    print "Template Phenotypic file for group analysis with repeated " \
          "measures (multiple scans) - %s" % file_name
    """

    # generate the group analysis subject lists
    file_name = os.path.join(data_config_outdir,
                             'participant_list_group_analysis_%s.txt'
                             % data_config_name)

    try:
        with open(file_name, 'w') as f:
            for sub in sorted(subID_set):
                print(sub, file=f)
    except:
        print('\n\nCPAC says: I couldn\'t save this file to your drive:\n')
        print(file_name, '\n\n')
        print('Make sure you have write access? Then come back. Don\'t ' \
              'worry.. I\'ll wait.\n\n')
        raise IOError

    print("Participant list required later for group analysis - %s\n\n" \
          % file_name)


def read_csv(csv_input):
    """
    Method to read csv file
     'Acquisition'
     'Reference'
     'Site'
     'TR (seconds)'

    """
    import csv
    from collections import defaultdict
    try:
        reader = csv.DictReader(open(csv_input, "U"))

        dict_labels = defaultdict(list)
        for line in reader:
            csv_dict = dict((k.lower(), v) for k, v in line.items())
            dict_labels[csv_dict.get('site')] = [
                csv_dict[key] for key in sorted(list(
                    csv_dict.keys()
                )) if key != 'site' and key != 'scan'
            ]

        if len(dict_labels) < 1:
            msg ="Scan Parameters File is either empty"\
                 "or missing header"
            logging.exception(msg)
            raise Exception(msg)

        return dict_labels

    except IOError:
        msg = "Error reading the csv file %s", csv_input
        logging.exception(msg)
        raise Exception(msg)
    except:
        msg = "Error reading scan parameters csv. Make sure you are using the correct template"
        logging.exception(msg)
        raise Exception(msg)


"""
Class to set dictionary keys as map attributes
"""
class Configuration(object):
    def __init__(self, config_map):
        for key in config_map:
            if config_map[key] == 'None':
                config_map[key] = None
            setattr(self, key, config_map[key])


def run(data_config):
    """
    Run method takes data_config
    file as the input argument
    """
    root = logging.getLogger()
    if root.handlers:
        for handler in root.handlers:
            root.removeHandler(handler)
    logging.basicConfig(filename=os.path.join(os.getcwd(), 'extract_data_logs.log'), filemode='w', level=logging.DEBUG,\
                    format="%(levelname)s %(asctime)s %(lineno)d %(message)s")

    print("For any errors or messages check the log file - %s"\
           % os.path.join(os.getcwd(), 'extract_data_logs.log'))

    c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), 'r')))

    if c.scanParametersCSV is not None:
        s_param_map = read_csv(c.scanParametersCSV)
    else:
        logging.debug("no scan parameters csv included\n"\
              "make sure you turn off slice timing correction option\n"\
              "in CPAC configuration\n")
        s_param_map = None

    generate_supplementary_files(c.outputSubjectListLocation, c.subjectListName)


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python extract_data.py data_config.yml")
        sys.exit()
    else:
        run(sys.argv[1])

Method to generate a CPAC input subject list     python file. The method extracts anatomical     and functional data for each site( if multiple site)     and/or scan parameters for each site and put it into     a data structure read by python      Example:     subjects_list =[        {         'subject_id' : '0050386',         'unique_id' : 'session_1',         'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',         'rest':{             'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',             'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',             }         'scan_parameters':{             'tr': '2',             'acquisition': 'alt+z2',             'reference': '17',             'first_tr': '',             'last_tr': '',             }         },     ]      or      subjects_list =[        {         'subject_id' : '0050386',         'unique_id' : 'session_1',         'anat': '/Users/home/data/NYU/0050386/session_1/anat_1/anat.nii.gz',         'rest':{             'rest_1_rest' : '/Users/home/data/NYU/0050386/session_1/rest_1/rest.nii.gz',             'rest_2_rest' : '/Users/home/data/NYU/0050386/session_1/rest_2/rest.nii.gz',             }           },     ] #method to read each line of the file into list #returns list #check if Template is correct #mapping between site and subject #check if subject is present in subject_list #method to split the input template path #into base, path before subject directory #and relative, path after subject directory #get anatomical base path and anatomical relative path #get functional base path, functional relative path and site-subject map #calculate the length of relative paths(path after subject directory) Method to check if there are sessions present #default #session present if path_length is equal to 3 Method to extract anatomical and functional         path for a session and print to file          Parameters         ----------         i : int             index of site         anat_sub : string             string containing subject/ concatenated             subject-session path for anatomical file         func_sub: string             string containing subject/ concatenated             subject-session path for functional file         session_id: string             session          Raises         ------         Exception #get anatomical file #iterate for each rest session #get scan_id Method which walks across each subject         path in the data site path          Parameters         ----------         index : int             index of site         sub : string             subject_id          Raises         ------         Exception #if there are sessions #check if subject is present in subject_list #check that subject is not in exclusion list Method to generate phenotypic template file     and subject list for group analysis # one of the participants in the subject list has no # functional scans # generate the phenotypic file templates for group analysis # generate the phenotypic file templates for repeated measures     if (len(session_set) > 1) and (len(scan_set) > 1):          file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \                 '_measures_mult_sessions_and_scans_%s.csv' \                 % data_config_name)          try:             f = open(file_name, 'wb')         except:             print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'             print file_name, '\n\n'             print 'Make sure you have write access? Then come back. Don\'t ' \                     'worry.. I\'ll wait.\n\n'             raise IOError          writer = csv.writer(f)         writer.writerow(['participant', 'session', 'series', 'EV1', '..'])          for session in sorted(session_set):             for scan in sorted(scan_set):                 for sub in sorted(subID_set):                     writer.writerow([sub, session, scan, ''])          f.close()          print "Template Phenotypic file for group analysis with repeated " \               "measures (multiple sessions and scans) - %s" % file_name      if (len(session_set) > 1):          file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \                 '_measures_multiple_sessions_%s.csv' % data_config_name)          try:             f = open(file_name, 'wb')         except:             print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'             print file_name, '\n\n'             print 'Make sure you have write access? Then come back. Don\'t ' \                   'worry.. I\'ll wait.\n\n'             raise IOError          writer = csv.writer(f)          writer.writerow(['participant', 'session', 'EV1', '..'])          for session in sorted(session_set):             for sub in sorted(subID_set):                 writer.writerow([sub, session, ''])          f.close()          print "Template Phenotypic file for group analysis with repeated " \               "measures (multiple sessions) - %s" % file_name      if (len(scan_set) > 1):          file_name = os.path.join(data_config_outdir, 'phenotypic_template_repeated' \                 '_measures_multiple_scans_%s.csv' % data_config_name)          try:             f = open(file_name, 'wb')         except:             print '\n\nCPAC says: I couldn\'t save this file to your drive:\n'             print file_name, '\n\n'             print 'Make sure you have write access? Then come back. Don\'t ' \                   'worry.. I\'ll wait.\n\n'             raise IOError          writer = csv.writer(f)          writer.writerow(['participant', 'series', 'EV1', '..'])          for scan in sorted(scan_set):             for sub in sorted(subID_set):                 writer.writerow([sub, scan, ''])          f.close()      print "Template Phenotypic file for group analysis with repeated " \           "measures (multiple scans) - %s" % file_name # generate the group analysis subject lists Method to read csv file      'Acquisition'      'Reference'      'Site'      'TR (seconds)' Class to set dictionary keys as map attributes Run method takes data_config     file as the input argument