def get_possible_templates():
    return POSSIBLE_TEMPLATES DEFAULT_TEMPLATE = 'GenericMNI' BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DEBUG = True
DOMAIN_NAME = "https://neurovault.org"
TEMPLATE_DEBUG = DEBUG
ADMINS = (
    (('Chris', 'krzysztof.gorgolewski@gmail.com'))
)
MANAGERS = ADMINS
SITE_ID = 1
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': 'postgres',
        # The following settings are not used with sqlite3:
        'USER': 'postgres',
        'HOST': 'db',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
        'PORT': '5432',        # Set to empty string for default.
    }
}
ALLOWED_HOSTS = ["*"]
TIME_ZONE = 'Europe/Berlin'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = True
MEDIA_ROOT = os.path.join(BASE_DIR,'media')
MEDIA_URL = '/public/media/'
PRIVATE_MEDIA_ROOT = '/var/www/image_data'
PRIVATE_MEDIA_URL = '/media/images'
STATIC_ROOT = '/var/www/static'
STATIC_URL = '/static/'
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'neurovault.apps.statmaps.middleware.CollectionRedirectMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
ROOT_URLCONF = 'neurovault.urls'
TEST_RUNNER = 'django.test.runner.DiscoverRunner'
WSGI_APPLICATION = 'neurovault.wsgi.application'
TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': (),
        'OPTIONS': {'context_processors': ("django.contrib.auth.context_processors.auth",
                                            "django.core.context_processors.debug",
                                            "django.core.context_processors.i18n",
                                            "django.core.context_processors.media",
                                            "django.core.context_processors.static",
                                            "django.core.context_processors.tz",
                                            "django.contrib.messages.context_processors.messages",
                                            'django.core.context_processors.request'),
                    'loaders': ('django.template.loaders.filesystem.Loader',
                                'django.template.loaders.app_directories.Loader',
                                )}
    }
]
INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'neurovault.apps.main',
    'neurovault.apps.statmaps',
    'neurovault.apps.users',
    'django.contrib.sitemaps',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
     #'django.contrib.admindocs',
    'social.apps.django_app.default',
    'rest_framework',
    'taggit',
    'crispy_forms',
    'taggit_templatetags',
    #'south',
    'dbbackup',
    'polymorphic',
    'djcelery',
    'django_cleanup',
    'file_resubmit',
    'django_mailgun',
    'django_hstore',
    'guardian',
    'oauth2_provider',
    'fixture_media',
    'raven.contrib.django.raven_compat',
)
AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'social.backends.facebook.FacebookOAuth2',
    'social.backends.google.GoogleOAuth2',
    'guardian.backends.ObjectPermissionBackend',
)
SOCIAL_AUTH_PIPELINE = (
    'social.pipeline.social_auth.social_details',
    'social.pipeline.social_auth.social_uid',
    'social.pipeline.social_auth.auth_allowed',
    'social.pipeline.social_auth.social_user',
    'social.pipeline.user.get_username',
    'social.pipeline.social_auth.associate_by_email',  # <--- enable this one
    'social.pipeline.user.create_user',
    'social.pipeline.social_auth.associate_user',
    'social.pipeline.social_auth.load_extra_data',
    'social.pipeline.user.user_details'
)
SOCIAL_AUTH_FACEBOOK_SCOPE = ['email']
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
REST_FRAMEWORK = {
    # Use hyperlinked styles by default.
    # Only used if the `serializer_class` attribute is not set on a view.
    'DEFAULT_MODEL_SERIALIZER_CLASS':
        'rest_framework.serializers.HyperlinkedModelSerializer',

    # LimitOffsetPagination will allow to set a ?limit= and ?offset=
    # variable in the URL.
    'DEFAULT_PAGINATION_CLASS':
        'neurovault.api.pagination.StandardResultPagination',

    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ],

    'DEFAULT_AUTHENTICATION_CLASSES': (
        'oauth2_provider.ext.rest_framework.OAuth2Authentication',
    ),
    'DEFAULT_RENDERER_CLASSES': (
        'neurovault.api.utils.ExplicitUnicodeJSONRenderer',
        'rest_framework.renderers.BrowsableAPIRenderer',
    ),
    'UNICODE_JSON': True,
}
OAUTH2_PROVIDER = {
    'REQUEST_APPROVAL_PROMPT': 'auto'
}
LOGIN_REDIRECT_URL = '/my_collections/'
CRISPY_TEMPLATE_PACK = 'bootstrap4'
DBBACKUP_STORAGE = 'dbbackup.storage.dropbox_storage'
DBBACKUP_TOKENS_FILEPATH = '/home/filo/dbtokens'
DBBACKUP_POSTGRES_BACKUP_COMMAND = 'export PGPASSWORD=neurovault\n pg_dump --username={adminuser} --host={host} --port={port} {databasename} >'
SENDFILE_BACKEND = 'sendfile.backends.development'
PRIVATE_MEDIA_REDIRECT_HEADER = 'X-Accel-Redirect'
PYCORTEX_DATASTORE = os.path.join(BASE_DIR,'pycortex_data')
CACHES = {
            'default': {
                'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
            },
            "file_resubmit": {
                'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
                "LOCATION": '/tmp/file_resubmit/'
            }
          }
EMAIL_BACKEND = 'django_mailgun.MailgunBackend'
MAILGUN_ACCESS_KEY = 'key-3ax6xnjp29jd6fds4gc373sgvjxteol0' # replace with a real key in production
MAILGUN_SERVER_NAME = 'samples.mailgun.org'# replace with 'neurovault.org' in production
DEFAULT_FROM_EMAIL = "noreply@neurovault.org"
    STATICFILES_DIRS = (
                        ('pycortex-resources', '/usr/local/lib/python2.7/site-packages/cortex/webgl/resources'),
                        ('pycortex-ctmcache', '/usr/local/share/pycortex/db/fsaverage/cache')
                        )
BROKER_URL = 'redis://redis:6379/0'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_DEFAULT_QUEUE = 'default'
CELERY_QUEUES = (
    Queue('default', Exchange('default'), routing_key='default'),
)
CELERY_IMPORTS = ('neurovault.apps.statmaps.tasks', )
CELERYBEAT_SCHEDULE = {
    'anima_crawl_every day': {
        'task': 'crawl_anima',
        'schedule': timedelta(days=1)
    },
}
CELERY_TIMEZONE = 'Europe/Berlin'
ANONYMOUS_USER_ID = -1
DEFAULT_OAUTH_APPLICATION_ID = -1
DEFAULT_OAUTH_APP_NAME = 'DefaultOAuthApp'
DEFAULT_OAUTH_APP_OWNER_ID = -2
DEFAULT_OAUTH_APP_OWNER_USERNAME = 'DefaultAppOwner'
OAUTH_PERSONAL_TOKEN_LENGTH = 40
    PRIVATE_MEDIA_ROOT = test_media_root
    CELERY_ALWAYS_EAGER = True
    CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
TAGGIT_CASE_INSENSITIVE=True
FIXTURE_DIRS = (
    'apps/statmaps/fixtures/',
)
MEDIA_ROOT = PRIVATE_MEDIA_ROOT def calculate_gene_expression_similarity(reduced_stat_map_data, mask="full"):
    store_file = "/ahba_data/store_max1_reduced.h5"
    subcortex_mask = "/ahba_data/subcortex_mask.npy"

    results_dfs = []
    with pd.HDFStore(store_file, 'r') as store:
        for donor_id in store.keys():
            print "Loading expression data (%s)" % donor_id
            expression_data = store.get(donor_id.replace(".", "_"))

            print "Getting statmap values (%s)" % donor_id
            nifti_values = reduced_stat_map_data[expression_data.columns]

            print "Removing missing values (%s)" % donor_id
            na_mask = np.isnan(nifti_values)
            if mask == "subcortex":
                na_mask = np.logical_or(na_mask,
                    np.isnan(np.load(subcortex_mask)[expression_data.columns]))
            elif mask == "cortex":
                na_mask = np.logical_or(na_mask, np.logical_not(np.isnan(
                    np.load(subcortex_mask)[expression_data.columns])))
            else:
                assert mask == "full"

            nifti_values = np.array(nifti_values)[np.logical_not(na_mask)]
            expression_data.drop(expression_data.columns[na_mask], axis=1, inplace=True)

            print "z scoring (%s)" % donor_id
            expression_data = pd.DataFrame(zscore(expression_data, axis=1), columns=expression_data.columns,
                                           index=expression_data.index)
            nifti_values = zscore(nifti_values)

            print "Calculating linear regressions (%s)" % donor_id
            regression_results = np.linalg.lstsq(np.c_[nifti_values, np.ones_like(nifti_values)], expression_data.T)
            results_df = pd.DataFrame({"slope": regression_results[0][0]}, index=expression_data.index)

            results_df.columns = pd.MultiIndex.from_tuples([(donor_id[1:], c,) for c in results_df.columns],
                                                           names=['donor_id', 'parameter'])

            results_dfs.append(results_df)

        print "Concatenating results"
        results_df = pd.concat(results_dfs, axis=1)
        del results_dfs

    t, p = ttest_1samp(results_df, 0.0, axis=1)
    group_results_df = pd.DataFrame({"t": t, "p": p}, columns=['t', 'p'], index=expression_data.index)
    _, group_results_df["p (FDR corrected)"], _, _ = multipletests(group_results_df.p, method='fdr_bh')
    group_results_df["variance explained (mean)"] = (results_df.xs('slope', axis=1, level=1) ** 2 * 100).mean(axis=1)
    group_results_df["variance explained (std)"] = (results_df.xs('slope', axis=1, level=1) ** 2 * 100).std(axis=1)
    del results_df
    probe_info = pd.read_csv("/ahba_data/probe_info_max1.csv", index_col=0).drop(['chromosome', "gene_id"], axis=1)
    group_results_df = group_results_df.join(probe_info)
    group_results_df = group_results_df[["gene_symbol", "entrez_id.1", "gene_name","t", "p", "p (FDR corrected)",
                                         "variance explained (mean)", "variance explained (std)"]]

    return group_results_df class CollectionForm(ModelForm):
    class Meta:
        exclude = ('owner', 'private_token', 'contributors', 'private')
        model = Collection
        widgets = {
            'communities': CheckboxSelectMultiple
        }

        # fieldsets = study_fieldsets
        # row_attrs = study_row_attrs

    def clean(self):
        cleaned_data = super(CollectionForm, self).clean()
        doi = self.cleaned_data['DOI']
        if doi.strip() == '':
            self.cleaned_data['DOI'] = None

        if self.cleaned_data['DOI']:
            self.cleaned_data['DOI'] = self.cleaned_data['DOI'].strip()
            try:
                self.cleaned_data["name"], self.cleaned_data["authors"], self.cleaned_data[
                    "paper_url"], _, self.cleaned_data["journal_name"] = get_paper_properties(self.cleaned_data['DOI'].strip())
            except:
                self._errors["DOI"] = self.error_class(
                    ["Could not resolve DOI"])
            else:
                if "name" in self._errors:
                    del self._errors["name"]
        elif "name" not in cleaned_data or not cleaned_data["name"]:
            self._errors["name"] = self.error_class(
                ["You need to set the name or the DOI"])
            self._errors["DOI"] = self.error_class(
                ["You need to set the name or the DOI"])

        return cleaned_data

    def __init__(self, *args, **kwargs):

        super(CollectionForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-10'
        self.helper.layout = Layout()
        tab_holder = TabHolder()
        for fs in collection_fieldsets:
            # manually enforce field exclusion
            fs[1]['fields'] = [
                v for v in fs[1]['fields'] if v not in self.Meta.exclude]
            tab_holder.append(Tab(fs[1]['legend'], *fs[1]['fields']))
        self.helper.layout.extend([tab_holder, Submit(
                                  'submit', 'Save', css_class="btn-large offset2")]) class UploadFileForm(Form):

    # TODO Need to upload in a temp directory
    # (upload_to="images/%s/%s"%(instance.collection.id, filename))
    file = FileField(required=False)

    def __init__(self, *args, **kwargs):
        super(UploadFileForm, self).__init__(*args, **kwargs)
        self.file = ''

    def clean(self):
        cleaned_data = super(UploadFileForm, self).clean()
        file = cleaned_data.get("file")
        if file:
            ext = os.path.splitext(file.name)[1]
            ext = ext.lower()
            if ext not in ['.zip', '.gz']:
                raise ValidationError("Not allowed filetype!") class SimplifiedStatisticMapForm(EditStatisticMapForm):

    class Meta(EditStatisticMapForm.Meta):
        fields = ('name', 'collection', 'description', 'map_type', 'modality', 'target_template_image', 'cognitive_paradigm_cogatlas',
                  'cognitive_contrast_cogatlas', 'cognitive_paradigm_description_url', 'file', 'ignore_file_warning', 'hdr_file', 'tags', 'is_thresholded',
                  'perc_bad_voxels') class NeuropowerStatisticMapForm(EditStatisticMapForm):
    def __init__(self, *args, **kwargs):
        super(NeuropowerStatisticMapForm, self).__init__(*args, **kwargs)
        self.fields['analysis_level'].required = True
        self.fields['number_of_subjects'].required = True

    class Meta(EditStatisticMapForm.Meta):
        fields = ('name', 'collection', 'description', 'map_type', 'modality', 'target_template_image', 'map_type','analysis_level','number_of_subjects','cognitive_paradigm_cogatlas',
                  'cognitive_contrast_cogatlas', 'cognitive_paradigm_description_url', 'file', 'ignore_file_warning', 'hdr_file', 'tags', 'is_thresholded',
                  'perc_bad_voxels') class StatisticMapForm(ImageForm):
    def __init__(self, *args, **kwargs):
        super(StatisticMapForm, self).__init__(*args, **kwargs)

    def clean(self, **kwargs):
        cleaned_data = super(StatisticMapForm, self).clean()
        django_file = cleaned_data.get("file")

        cleaned_data["is_valid"] = True #This will be only saved if the form will validate
        cleaned_data["tags"] = clean_tags(cleaned_data)
        # print cleaned_data

        if "analysis_level" in cleaned_data.keys():
            if cleaned_data.get('analysis_level') == 'S':
                cleaned_data['number_of_subjects'] = 1
                if 'number_of_subjects' in self._errors:
                    del self._errors['number_of_subjects']

        if "data_origin" in cleaned_data.keys() and cleaned_data["data_origin"] == "surface":
            cleaned_data["is_thresholded"] = False
            cleaned_data["not_mni"] = False
            cleaned_data["perc_bad_voxels"] = 0
            cleaned_data["brain_coverage"] = 100
        elif django_file and "file" not in self._errors and "hdr_file" not in self._errors:
            django_file.open()
            fileobj = StringIO(django_file.read())
            django_file.seek(0)
            gzfileobj = GzipFile(
                filename=django_file.name, mode='rb', fileobj=fileobj)
            nii = nb.Nifti1Image.from_file_map(
                {'image': nb.FileHolder(django_file.name, gzfileobj)})
            cleaned_data["is_thresholded"], ratio_bad = is_thresholded(nii)
            cleaned_data["perc_bad_voxels"] = ratio_bad*100.0

            if cleaned_data["is_thresholded"] and not cleaned_data.get("ignore_file_warning") and cleaned_data.get("map_type") != "R":
                self._errors["file"] = self.error_class(
                    ["This map seems to be thresholded (%.4g%% of voxels are zeros). Please use an unthresholded version of the map if possible." % (cleaned_data["perc_bad_voxels"])])
                if cleaned_data.get("hdr_file"):
                    self._errors["hdr_file"] = self.error_class(
                        ["This map seems to be thresholded (%.4g%% of voxels are zeros). Please use an unthresholded version of the map if possible." % (cleaned_data["perc_bad_voxels"])])
                self.fields[
                    "ignore_file_warning"].widget = forms.CheckboxInput()
            else:
                cleaned_data["not_mni"], cleaned_data["brain_coverage"], cleaned_data[
                    "perc_voxels_outside"] = not_in_mni(nii, target_template_image=cleaned_data["target_template_image"])
                if cleaned_data["not_mni"] and not cleaned_data.get("ignore_file_warning") and cleaned_data.get(
                    "map_type") != "R":
                    self._errors["file"] = self.error_class(
                        ["This map seems not to be in the MNI space (%.4g%% of meaningful voxels are outside of the brain). Please use transform your data to MNI space." % (cleaned_data["perc_voxels_outside"])])
                    if cleaned_data.get("hdr_file"):
                        self._errors["hdr_file"] = self.error_class(
                            ["This map seems not to be in the MNI space (%.4g%% of meaningful voxels are outside of the brain). Please use transform your data to MNI space." % (cleaned_data["perc_voxels_outside"])])
                    self.fields[
                        "ignore_file_warning"].widget = forms.CheckboxInput()

            if cleaned_data.get("map_type") == "R":
                if "not_mni" in cleaned_data:
                    del cleaned_data["not_mni"]
                if "is_thresholded" in cleaned_data:
                    del cleaned_data["is_thresholded"]

        return cleaned_data

    class Meta(ImageForm.Meta):
        model = StatisticMap
        fields = ('name', 'collection', 'description', 'map_type', 'modality',
                  'target_template_image', 'cognitive_paradigm_cogatlas',
                  'cognitive_contrast_cogatlas', 'cognitive_paradigm_description_url',
                  'analysis_level', 'number_of_subjects', 'contrast_definition', 'figure',
                  'age', 'gender', 'ethnicity', 'race', 'handedness',   'bis11_score', 'bis_bas_score',
                  'spsrq_score', 'BMI', 'fat_percentage', 'waist_hip_ratio', 'hours_since_last_meal', 'days_since_menstruation',
                  'mean_PDS_score',  'tanner_stage',
                  'file', 'ignore_file_warning', 'hdr_file', 'tags', 'statistic_parameters',
                  'smoothness_fwhm', 'is_thresholded', 'perc_bad_voxels', 'is_valid', 'data_origin')
        widgets = {
            'file': AdminResubmitFileWidget,
            'hdr_file': AdminResubmitFileWidget,
            'is_thresholded': HiddenInput,
            'ignore_file_warning': HiddenInput,
            'perc_bad_voxels': HiddenInput,
            'not_mni': HiddenInput,
            'brain_coverage': HiddenInput,
            'perc_voxels_outside': HiddenInput,
            'is_valid': HiddenInput,
            'data_origin': HiddenInput
        }

    def save_afni_slices(self, commit):
        try:
            orig_img = self.instance

            for n, (label, brick) in enumerate(self.afni_subbricks):
                brick_fname = os.path.split(brick)[-1]
                mfile = memory_uploadfile(brick, brick_fname, orig_img.file)
                brick_img = StatisticMap(name='%s - %s' % (orig_img.name, label), collection=orig_img.collection,
                                         file=mfile)
                for field in set(self.Meta.fields) - set(['file', 'hdr_file', 'name', 'collection']):
                    if field in self.cleaned_data:
                        setattr(brick_img, field, self.cleaned_data[field])

                brick_img.save()
            return orig_img.collection

        finally:
            try:
                shutil.rmtree(self.afni_tmp)
            except OSError as exc:
                if exc.errno != 2:
                    raise

    def save(self, commit=True):
        if self.afni_subbricks:
            return self.save_afni_slices(commit)
        else:
            return super(StatisticMapForm, self).save(commit=commit) class EditStatisticMapForm(StatisticMapForm):
    def __init__(self, *args, **kwargs):
        user = kwargs['user']
        del kwargs['user']
        super(EditStatisticMapForm, self).__init__(*args, **kwargs)
        if user.is_superuser:
            self.fields['collection'].queryset = Collection.objects.all()
        else:
            self.fields['collection'].queryset = get_objects_for_user(
                user, 'statmaps.change_collection') class OwnerCollectionForm(CollectionForm):
    contributors = ContributorCommaField(
        queryset=None, required=False, help_text="Select other NeuroVault users to add as contributes to the collection.  Contributors can add, edit and delete images in the collection.")

    class Meta:
        exclude = ('owner', 'private_token')
        model = Collection
        widgets = {
            'private': forms.RadioSelect,
            'communities': CheckboxSelectMultiple
        }

    def __init__(self, *args, **kwargs):
        super(OwnerCollectionForm, self).__init__(*args, **kwargs)
        self.fields['contributors'].queryset = User.objects.exclude(
            pk=self.instance.owner.pk) class EditAtlasForm(AtlasForm):

    def __init__(self, *args, **kwargs):
        user = kwargs['user']
        del kwargs['user']
        super(EditAtlasForm, self).__init__(*args, **kwargs)
        if user.is_superuser:
            self.fields['collection'].queryset = Collection.objects.all()
        else:
            self.fields['collection'].queryset = get_objects_for_user(
                user, 'statmaps.change_collection')

    class Meta(AtlasForm.Meta):
        exclude = () class AtlasForm(ImageForm):

    class Meta(ImageForm.Meta):
        model = Atlas
        fields = ('name', 'collection', 'description', 'figure',
                  'file', 'hdr_file', 'label_description_file', 'tags') class EditNIDMResultStatisticMapForm(NIDMResultStatisticMapForm):

    def __init__(self, user, *args, **kwargs):
        super(EditNIDMResultStatisticMapForm, self).__init__(*args, **kwargs) class NIDMResultsForm(forms.ModelForm, NIDMResultsValidationMixin):

    class Meta:
        model = NIDMResults
        widgets = {
            'is_valid': forms.HiddenInput()
        }
        exclude = []

    def __init__(self, *args, **kwargs):
        super(NIDMResultsForm, self).__init__(*args, **kwargs)

        for fld in ['ttl_file']:
            if self.instance.pk is None:
                self.fields[fld].widget = HiddenInput()
            else:
                self.fields[fld].widget = PathOnlyWidget()

        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-10'
        self.helper.form_tag = True
        self.helper.add_input(Submit('submit', 'Submit'))
        self.helper.add_input(
            Button('delete', 'Delete',
                   onclick='window.location.href=window.location.href+"/delete"'))
        self.nidm = None
        self.new_statmaps = []

        if self.instance.pk is not None:
            self.fields['name'].widget = HiddenInput()
            if self.fields.get('collection'):
                self.fields['collection'].widget = HiddenInput()

    def clean(self):
        cleaned_data = super(NIDMResultsForm, self).clean()
        cleaned_data["tags"] = clean_tags(cleaned_data)
        # only process new uploads or replaced zips
        if self.instance.pk is None or 'zip_file' in self.changed_data:
            self.cleaned_data = self.clean_and_validate(cleaned_data)

    def save(self, commit=True):
        if self.instance.pk is None or 'zip_file' in self.changed_data:
            do_update = True

        nidm_r = super(NIDMResultsForm, self).save(commit)
        if commit and do_update is not None:
            self.save_nidm()
            self.update_ttl_urls()
        return nidm_r

    def update_ttl_urls(self):
        handle_update_ttl_urls(self.instance)

    def save_nidm(self):
        if self.nidm and 'zip_file' in self.changed_data:
            save_nidm_statmaps(self.nidm, self.instance)
            # todo: rewrite ttl class NIDMViewForm(forms.ModelForm):

    class Meta:
        model = NIDMResults
        exclude = ['is_valid']

    def __init__(self, *args, **kwargs):
        super(NIDMViewForm, self).__init__(*args, **kwargs)

        for fld in ['ttl_file', 'zip_file']:
            self.fields[fld].widget = PathOnlyWidget()
        for fld in self.fields:
            self.fields[fld].widget.attrs['readonly'] = 'readonly'
        self.fields['name'].widget = HiddenInput()
        if self.fields.get('collection'):
            self.fields['collection'].widget = HiddenInput()

        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-10'
        self.helper.form_tag = True class AddStatisticMapForm(StatisticMapForm):

    class Meta(StatisticMapForm.Meta):
        fields = ('name', 'description', 'map_type', 'modality', 'target_template_image', 'cognitive_paradigm_cogatlas',
                  'cognitive_contrast_cogatlas',
                  'file', 'ignore_file_warning', 'hdr_file', 'surface_left_file', 'surface_right_file',
                  'cognitive_paradigm_description_url', 'analysis_level', 'number_of_subjects',
                  'contrast_definition', 'figure',
                  'age', 'gender', 'ethnicity', 'race', 'handedness', 'bis11_score',
                  'bis_bas_score',
                  'spsrq_score', 'BMI', 'fat_percentage', 'waist_hip_ratio',
                  'hours_since_last_meal', 'days_since_menstruation',
                  'mean_PDS_score', 'tanner_stage',
                  'tags', 'statistic_parameters',
                  'smoothness_fwhm', 'is_thresholded', 'perc_bad_voxels', 'data_origin') class MetaanalysisForm(ModelForm):

    class Meta:
        exclude = ('owner', 'maps', 'output_maps', 'status')
        model = Metaanalysis
        # fieldsets = study_fieldsets
        # row_attrs = study_row_attrs

    def __init__(self, *args, **kwargs):

        super(MetaanalysisForm, self).__init__(*args, **kwargs)

        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-10'
        self.helper.add_input(Submit('submit', 'Submit')) class Collection(models.Model):
    name = models.CharField(max_length=200, unique = True, null=False, verbose_name="Name of collection")
    DOI = models.CharField(max_length=200, unique=True, blank=True, null=True, default=None, verbose_name="DOI of the corresponding paper", help_text="Required if you want your maps to be archived in Stanford Digital Repository")
    authors = models.CharField(max_length=5000, blank=True, null=True)
    paper_url = models.CharField(max_length=200, blank=True, null=True)
    journal_name = models.CharField(max_length=200, blank=True, null=True, default=None)
    description = models.TextField(blank=True, null=True)
    full_dataset_url = models.URLField(max_length=200, blank=True, null=True, verbose_name="Full dataset URL", help_text="Link to an external dataset the maps in this collection have been generated from (for example: \"https://openfmri.org/dataset/ds000001\" or \"http://dx.doi.org/10.15387/fcp_indi.corr.mpg1\")")
    owner = models.ForeignKey(User)
    contributors = models.ManyToManyField(User,related_name="collection_contributors",related_query_name="contributor", blank=True,help_text="Select other NeuroVault users to add as contributes to the collection.  Contributors can add, edit and delete images in the collection.",verbose_name="Contributors")
    private = models.BooleanField(choices=((False, 'Public (The collection will be accessible by anyone and all the data in it will be distributed under CC0 license)'),
                                           (True, 'Private (The collection will be not listed in the NeuroVault index. It will be possible to shared it with others at a private URL.)')), default=False,verbose_name="Accessibility")
    private_token = models.CharField(max_length=8,blank=True,null=True,unique=True,db_index=True, default=None)
    add_date = models.DateTimeField('date published', auto_now_add=True)
    modify_date = models.DateTimeField('date modified', auto_now=True)
    doi_add_date = models.DateTimeField('date the DOI was added', editable=False, blank=True, null=True, db_index=True)
    communities = models.ManyToManyField(Community, related_name="collections",
                                         related_query_name="collection", blank=True,
                                         help_text="Is this collection part of any special Community?",
                                         verbose_name="Communities", default=None)

    type_of_design = models.CharField(choices=[('blocked', 'blocked'), ('eventrelated', 'event_related'), ('hybridblockevent', 'hybrid block/event'), ('other', 'other')], max_length=200, blank=True, help_text="Blocked, event-related, hybrid, or other", null=True, verbose_name="Type of design")
    number_of_imaging_runs = models.IntegerField(help_text="Number of imaging runs acquired", null=True, verbose_name="No. of imaging runs", blank=True)
    number_of_experimental_units = models.IntegerField(help_text="Number of blocks, trials or experimental units per imaging run", null=True, verbose_name="No. of experimental units", blank=True)
    length_of_runs = models.FloatField(help_text="Length of each imaging run in seconds", null=True, verbose_name="Length of runs", blank=True)
    length_of_blocks = models.FloatField(help_text="For blocked designs, length of blocks in seconds", null=True, verbose_name="Length of blocks", blank=True)
    length_of_trials = models.CharField(help_text="Length of individual trials in seconds. If length varies across trials, enter 'variable'. ", max_length=200, null=True, verbose_name="Length of trials", blank=True)
    optimization = models.NullBooleanField(help_text="Was the design optimized for efficiency", null=True, verbose_name="Optimization?", blank=True)
    optimization_method = models.CharField(help_text="What method was used for optimization?", verbose_name="Optimization method", max_length=200, null=True, blank=True)
    subject_age_mean = models.FloatField(help_text="Mean age of subjects", null=True, verbose_name="Subject age mean", blank=True)
    subject_age_min = models.FloatField(help_text="Minimum age of subjects", null=True, verbose_name="Subject age min", blank=True)
    subject_age_max = models.FloatField(help_text="Maximum age of subjects", null=True, verbose_name="Subject age max", blank=True)
    handedness = models.CharField(choices=[('right', 'right'), ('left', 'left'), ('both', 'both')], max_length=200, blank=True, help_text="Handedness of subjects", null=True, verbose_name="Handedness")
    proportion_male_subjects = models.FloatField(validators =[MinValueValidator(0.0), MaxValueValidator(1.0)], help_text="The proportion (not percentage) of subjects who were male", null=True, verbose_name="Prop. male subjects", blank=True)
    inclusion_exclusion_criteria = models.CharField(help_text="Additional inclusion/exclusion criteria, if any (including specific sampling strategies that limit inclusion to a specific group, such as laboratory members)", verbose_name="Inclusion / exclusion criteria", max_length=200, null=True, blank=True)
    number_of_rejected_subjects = models.IntegerField(help_text="Number of subjects scanned but rejected from analysis", null=True, verbose_name="No. of rejected subjects", blank=True)
    group_comparison = models.NullBooleanField(help_text="Was this study a comparison between subject groups?", null=True, verbose_name="Group comparison?", blank=True)
    group_description = models.CharField(help_text="A description of the groups being compared", verbose_name="Group description", max_length=200, null=True, blank=True)
    scanner_make = models.CharField(help_text="Manufacturer of MRI scanner", verbose_name="Scanner make", max_length=200, null=True, blank=True)
    scanner_model = models.CharField(help_text="Model of MRI scanner", verbose_name="Scanner model", max_length=200, null=True, blank=True)
    field_strength = models.FloatField(help_text="Field strength of MRI scanner (in Tesla)", null=True, verbose_name="Field strength", blank=True)
    pulse_sequence = models.CharField(help_text="Description of pulse sequence used for fMRI", verbose_name="Pulse sequence", max_length=200, null=True, blank=True)
    parallel_imaging = models.CharField(help_text="Description of parallel imaging method and parameters", verbose_name="Parallel imaging", max_length=200, null=True, blank=True)
    field_of_view = models.FloatField(help_text="Imaging field of view in millimeters", null=True, verbose_name="Field of view", blank=True)
    matrix_size = models.IntegerField(help_text="Matrix size for MRI acquisition", null=True, verbose_name="Matrix size", blank=True)
    slice_thickness = models.FloatField(help_text="Distance between slices (includes skip or distance factor) in millimeters", null=True, verbose_name="Slice thickness", blank=True)
    skip_distance = models.FloatField(help_text="The size of the skipped area between slices in millimeters", null=True, verbose_name="Skip distance", blank=True)
    acquisition_orientation = models.CharField(help_text="The orientation of slices", verbose_name="Acquisition orientation", max_length=200, null=True, blank=True)
    order_of_acquisition = models.CharField(choices=[('ascending', 'ascending'), ('descending', 'descending'), ('interleaved', 'interleaved')], max_length=200, blank=True, help_text="Order of acquisition of slices (ascending, descending, or interleaved)", null=True, verbose_name="Order of acquisition")
    repetition_time = models.FloatField(help_text="Repetition time (TR) in milliseconds", null=True, verbose_name="Repetition time", blank=True)
    echo_time = models.FloatField(help_text="Echo time (TE) in milliseconds", null=True, verbose_name="Echo time", blank=True)
    flip_angle = models.FloatField(help_text="Flip angle in degrees", null=True, verbose_name="Flip angle", blank=True)
    software_package = models.CharField(help_text="If a single software package was used for all analyses, specify that here", verbose_name="Software package", max_length=200, null=True, blank=True)
    software_version = models.CharField(help_text="Version of software package used", verbose_name="Software version", max_length=200, null=True, blank=True)
    order_of_preprocessing_operations = models.CharField(help_text="Specify order of preprocessing operations", verbose_name="Order of preprocessing", max_length=200, null=True, blank=True)
    quality_control = models.CharField(help_text="Describe quality control measures", verbose_name="Quality control", max_length=200, null=True, blank=True)
    used_b0_unwarping = models.NullBooleanField(help_text="Was B0 distortion correction used?", null=True, verbose_name="Used B0 unwarping?", blank=True)
    b0_unwarping_software = models.CharField(help_text="Specify software used for distortion correction if different from the main package", verbose_name="B0 unwarping software", max_length=200, null=True, blank=True)
    used_slice_timing_correction = models.NullBooleanField(help_text="Was slice timing correction used?", null=True, verbose_name="Slice timing correction?", blank=True)
    slice_timing_correction_software = models.CharField(help_text="Specify software used for slice timing correction if different from the main package", verbose_name="Slice timing correction software", max_length=200, null=True, blank=True)
    used_motion_correction = models.NullBooleanField(help_text="Was motion correction used?", null=True, verbose_name="Motion correction?", blank=True)
    motion_correction_software = models.CharField(help_text="Specify software used for motion correction if different from the main package", verbose_name="Motion correction software", max_length=200, null=True, blank=True)
    motion_correction_reference = models.CharField(help_text="Reference scan used for motion correction", verbose_name="Motion correction reference", max_length=200, null=True, blank=True)
    motion_correction_metric = models.CharField(help_text="Similarity metric used for motion correction", verbose_name="Motion correction metric", max_length=200, null=True, blank=True)
    motion_correction_interpolation = models.CharField(help_text="Interpolation method used for motion correction", verbose_name="Motion correction interpolation", max_length=200, null=True, blank=True)
    used_motion_susceptibiity_correction = models.NullBooleanField(help_text="Was motion-susceptibility correction used?", null=True, verbose_name="Motion susceptibility correction?", blank=True)
    used_intersubject_registration = models.NullBooleanField(help_text="Were subjects registered to a common stereotactic space?", null=True, verbose_name="Intersubject registration?", blank=True)
    intersubject_registration_software = models.CharField(help_text="Specify software used for intersubject registration if different from main package", verbose_name="Registration software", max_length=200, null=True, blank=True)
    intersubject_transformation_type = models.CharField(choices=[('linear', 'linear'), ('nonlinear', 'nonlinear')], max_length=200, blank=True, help_text="Was linear or nonlinear registration used?", null=True, verbose_name="Intersubject transformation type")
    nonlinear_transform_type = models.CharField(help_text="If nonlinear registration was used, describe transform method", verbose_name="Nonlinear transform type", max_length=200, null=True, blank=True)
    transform_similarity_metric = models.CharField(help_text="Similarity metric used for intersubject registration", verbose_name="Transform similarity metric", max_length=200, null=True, blank=True)
    interpolation_method = models.CharField(help_text="Interpolation method used for intersubject registration", verbose_name="Interpolation method", max_length=200, null=True, blank=True)
    object_image_type = models.CharField(help_text="What type of image was used to determine the transformation to the atlas? (e.g. T1, T2, EPI)", verbose_name="Object image type", max_length=200, null=True, blank=True)
    functional_coregistered_to_structural = models.NullBooleanField(help_text="Were the functional images coregistered to the subject's structural image?", null=True, verbose_name="Coregistered to structural?", blank=True)
    functional_coregistration_method = models.CharField(help_text="Method used to coregister functional to structural images", verbose_name="Coregistration method", max_length=200, null=True, blank=True)
    coordinate_space = models.CharField(choices=[('mni', 'MNI'), ('talairach', 'Talairach'), ('mni2tal', 'MNI2Tal'), ('other', 'other')], max_length=200, blank=True, help_text="Name of coordinate space for registration target", null=True, verbose_name="Coordinate space")
    target_template_image = models.CharField(help_text="Name of target template image", verbose_name="Target template image", max_length=200, null=True, blank=True)
    target_resolution = models.FloatField(help_text="Voxel size of target template in millimeters", null=True, verbose_name="Target resolution", blank=True)
    used_smoothing = models.NullBooleanField(help_text="Was spatial smoothing applied?", null=True, verbose_name="Used smoothing?", blank=True)
    smoothing_type = models.CharField(help_text="Describe the type of smoothing applied", verbose_name="Type of smoothing", max_length=200, null=True, blank=True)
    smoothing_fwhm = models.FloatField(help_text="The full-width at half-maximum of the smoothing kernel in millimeters", null=True, verbose_name="Smoothing FWHM", blank=True)
    resampled_voxel_size = models.FloatField(help_text="Voxel size in mm of the resampled, atlas-space images", null=True, verbose_name="Resampled voxel size", blank=True)
    intrasubject_model_type = models.CharField(help_text="Type of model used (e.g., regression)", verbose_name="Model type", max_length=200, null=True, blank=True)
    intrasubject_estimation_type = models.CharField(help_text="Estimation method used for model (e.g., OLS, generalized least squares)", verbose_name="Estimation type", max_length=200, null=True, blank=True)
    intrasubject_modeling_software = models.CharField(help_text="Software used for intrasubject modeling if different from overall package", verbose_name="Modeling software", max_length=200, null=True, blank=True)
    hemodynamic_response_function = models.CharField(help_text="Nature of HRF model", verbose_name="Hemodynamic response function", max_length=200, null=True, blank=True)
    used_temporal_derivatives = models.NullBooleanField(help_text="Were temporal derivatives included?", null=True, verbose_name="Temporal derivatives?", blank=True)
    used_dispersion_derivatives = models.NullBooleanField(help_text="Were dispersion derivatives included?", null=True, verbose_name="Dispersion derivatives?", blank=True)
    used_motion_regressors = models.NullBooleanField(help_text="Were motion regressors included?", null=True, verbose_name="Motion regressors?", blank=True)
    used_reaction_time_regressor = models.NullBooleanField(help_text="Was a reaction time regressor included?", null=True, verbose_name="Reaction time regressor?", blank=True)
    used_orthogonalization = models.NullBooleanField(help_text="Were any regressors specifically orthogonalized with respect to others?", null=True, verbose_name="Orthogonalization?", blank=True)
    orthogonalization_description = models.CharField(help_text="If orthogonalization was used, describe here", verbose_name="Orthogonalization description", max_length=200, null=True, blank=True)
    used_high_pass_filter = models.NullBooleanField(help_text="Was high pass filtering applied?", null=True, verbose_name="High-pass filter?", blank=True)
    high_pass_filter_method = models.CharField(help_text="Describe method used for high pass filtering", verbose_name="High-pass filtering method", max_length=200, null=True, blank=True)
    autocorrelation_model = models.CharField(help_text="What autocorrelation model was used (or 'none' of none was used)", verbose_name="Autocorrelation method", max_length=200, null=True, blank=True)
    group_model_type = models.CharField(help_text="Type of group model used (e.g., regression)", verbose_name="Group model type", max_length=200, null=True, blank=True)
    group_estimation_type = models.CharField(help_text="Estimation method used for group model (e.g., OLS, generalized least squares)", verbose_name="Group estimation type", max_length=200, null=True, blank=True)
    group_modeling_software = models.CharField(help_text="Software used for group modeling if different from overall package", verbose_name="Group modeling software", max_length=200, null=True, blank=True)
    group_inference_type = models.CharField(choices=[('randommixedeffects', 'random/mixed effects'), ('fixedeffects', 'fixed effects')], max_length=200, blank=True, help_text="Type of inference for group model", null=True, verbose_name="Group inference type")
    group_model_multilevel = models.CharField(help_text="If more than 2-levels, describe the levels and assumptions of the model (e.g. are variances assumed equal between groups)", verbose_name="Multilevel modeling", max_length=200, null=True, blank=True)
    group_repeated_measures = models.NullBooleanField(help_text="Was this a repeated measures design at the group level?", null=True, verbose_name="Repeated measures", blank=True)
    group_repeated_measures_method = models.CharField(help_text="If multiple measurements per subject, list method to account for within subject correlation, exact assumptions made about correlation/variance", verbose_name="Repeated measures method", max_length=200, null=True, blank=True)
    nutbrain_hunger_state = models.CharField(choices=[('I', 'Fed (<1h after meal)'),
                                                      ('II', '2-3 h fasted'),
                                                      ('III', '4-6 h fasted'),
                                                      ('IV', '7-9h fasted'),
                                                      ('V', 'fasted overnight (> 10h)'),
                                                      ('VI', '36h fast')], max_length=200,
                                             blank=True, null=True,
                                             verbose_name="Hunger state")
    nutbrain_food_viewing_conditions = models.CharField(max_length=200,
                                                        blank=True,
                                                        null=True,
                                                        verbose_name="Food viewing conditions",
                                                        help_text="Image categories")
    nutbrain_food_choice_type = models.CharField(max_length=200,
                                                blank=True,
                                                null=True,
                                                verbose_name="Food choice type",
                                                help_text="Choice conditions/image types")
    nutbrain_taste_conditions = models.CharField(max_length=200,
                                                 blank=True,
                                                 null=True,
                                                 verbose_name="Taste conditions")
    nutbrain_odor_conditions = models.CharField(max_length=200,
                                                 blank=True,
                                                 null=True,
                                                 verbose_name="Odor conditions")


    @property
    def is_statisticmap_set(self):
        return all((isinstance(i, StatisticMap) for i in self.basecollectionitem_set.all()))

    def get_absolute_url(self):
        return_cid = self.id
        if self.private:
            return_cid = self.private_token
        return reverse('collection_details', args=[str(return_cid)])

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if self.DOI is not None and self.DOI.strip() == "":
            self.DOI = None
        if self.private_token is not None and self.private_token.strip() == "":
            self.private_token = None

        if self.DOI and not self.private and not self.doi_add_date:
            self.doi_add_date = datetime.now()

        # run calculations when collection turns public
        privacy_changed = False
        DOI_changed = False
        if self.pk is not None:
            old_object = Collection.objects.get(pk=self.pk)
            old_is_private = old_object.private
            old_has_DOI = old_object.DOI is not None
            privacy_changed = old_is_private != self.private
            DOI_changed = old_has_DOI != (self.DOI is not None)

        super(Collection, self).save(*args, **kwargs)

        if (privacy_changed and not self.private) or (DOI_changed and self.DOI is not None):
            for image in self.basecollectionitem_set.instance_of(Image).all():
                if image.pk:
                    run_voxelwise_pearson_similarity.apply_async([image.pk])

    class Meta:
        app_label = 'statmaps'

    def delete(self, using=None):
        cid = self.pk
        for image in self.basecollectionitem_set.instance_of(Image):
            image.delete()
        for nidmresult in self.basecollectionitem_set.instance_of(NIDMResults):
            nidmresult.delete()
        ret = super(Collection, self).delete(using=using)
        collDir = os.path.join(PRIVATE_MEDIA_ROOT, 'images',str(cid))
        try:
            shutil.rmtree(collDir)
        except OSError:
            print 'Image directory for collection %s does not exist' %cid

        return ret class Image(BaseCollectionItem):
    file = models.FileField(upload_to=upload_img_to, null=False, blank=False, storage=DoubleExtensionStorage(), verbose_name='File with the unthresholded volume map (.img, .nii, .nii.gz)')
    surface_left_file = models.FileField(upload_to=upload_img_to, null=True, blank=True, storage=DoubleExtensionStorage(), verbose_name='File with the unthresholded LEFT hemisphere fsaverage surface map (.mgh, .curv, .gii)')
    surface_right_file = models.FileField(upload_to=upload_img_to, null=True, blank=True, storage=DoubleExtensionStorage(), verbose_name='File with the unthresholded RIGHT hemisphere fsaverage surface map (.mgh, .curv, .gii)')
    data_origin = models.CharField(
                    help_text=("Was this map originaly derived from volume or surface?"),
                    verbose_name="Data origin",
                    default='volume',
                    max_length=200, null=True, blank=True, choices=[('volume','volume'), ('surface', 'surface')])
    target_template_image = models.CharField(
        choices=get_target_template_list(),
        help_text="Name of target template image",
        verbose_name="Target template image",
        default=DEFAULT_TEMPLATE, max_length=200, null=False, blank=False)
    subject_species = models.CharField(max_length=200, default=POSSIBLE_TEMPLATES[DEFAULT_TEMPLATE]['species'], blank=True, null=True)
    figure = models.CharField(help_text="Which figure in the corresponding paper was this map displayed in?", verbose_name="Corresponding figure", max_length=200, null=True, blank=True)

    ### Added for the NutBrain project BEGIN
    handedness = models.CharField(max_length=200, null=True, blank=True, choices=[('L', 'Left'),
                                                                                  ('R', 'Right')],
                                  verbose_name="Handedness")
    age = models.FloatField(null=True, blank=True, verbose_name="Age (years)")
    gender = models.CharField(max_length=200, null=True, blank=True,
                              choices=[('M', 'Male'),
                                       ('F', 'Female'),
                                       ('O', 'Other')],
                              verbose_name="Gender")
    race = models.CharField(max_length=200, null=True, blank=True,
                            choices=[('W', 'White'),
                                     ('B', 'Black or African American'),
                                     ('I', 'American Indian or Alaska Native'),
                                     ('A', 'Asian'),
                                     ('H', 'Native Hawaiian or Other Pacific Islander')],
                            verbose_name="Race (US Census definition)")
    ethnicity = models.CharField(max_length=200, null=True, blank=True,
                                 choices=[('H', 'Hispanic or Latino'),
                                          ('NH', 'Not Hispanic or Latino')],
                                 verbose_name="Ethnicity (US Census definition)")
    BMI = models.FloatField(null=True, blank=True, verbose_name="Body Mass Index (kg/m2)")
    fat_percentage = models.FloatField(null=True, blank=True, verbose_name="% body fat")
    waist_hip_ratio = models.FloatField(null=True, blank=True, verbose_name="waist-hip-ratio")
    mean_PDS_score = models.FloatField(null=True, blank=True,
                                       verbose_name="Mean Puberty Development Scale score")
    tanner_stage = models.CharField(max_length=200, null=True, blank=True,
                                    choices=[('I', 'I'),
                                             ('II', 'II'),
                                             ('III', 'III'),
                                             ('IV', 'IV'),
                                             ('V', 'V')],
                                    verbose_name="Tanner stage")
    days_since_menstruation = models.FloatField(null=True, blank=True,
                                                verbose_name="Number of days since menstruation")
    hours_since_last_meal = models.FloatField(null=True, blank=True,
                                              verbose_name="Time since last meal (hours)")
    bis_bas_score = models.FloatField(null=True, blank=True,
                                      verbose_name="Behavioral inhibition, behavioral activation "
                                                   "(BIS/BAS) score")
    spsrq_score = models.FloatField(null=True, blank=True,
                                    verbose_name="Sensitivity to Punishment and Sensitivity to "
                                                 "Reward Questionnaire (SPSRQ) score")
    bis11_score = models.FloatField(null=True, blank=True,
                                    verbose_name="Barratt Impulsiveness Scale (BIS-11) score")
    # END


    thumbnail = models.FileField(help_text="The orthogonal view thumbnail path of the nifti image",
                                 null=True, blank=True, upload_to=upload_img_to,
                                 verbose_name='Image orthogonal view thumbnail 2D bitmap',
                                 storage=DoubleExtensionStorage())
    reduced_representation = models.FileField(help_text=("Binary file with the vector of in brain values resampled to lower resolution"),
                                              verbose_name="Reduced representation of the image",
                                              null=True, blank=True, upload_to=upload_img_to,
                                              storage=OverwriteStorage())
    data = hstore.DictionaryField(blank=True, null=True)
    hstore_objects = hstore.HStoreManager()


    def get_absolute_url(self):
        return_args = [str(self.id)]
        url_name = 'image_details'
        if self.collection.private:
            return_args.insert(0,str(self.collection.private_token))
            url_name = 'private_image_details'
        return reverse(url_name, args=return_args)

    def get_thumbnail_url(self):
        try:
            url =  self.thumbnail.url
        except ValueError:
            url = os.path.abspath(os.path.join("/static","images","glass_brain_empty.jpg"))
        return url

    @classmethod
    def create(cls, my_file, my_file_name, my_name, my_desc, my_collection_pk, my_map_type):
        my_collection = Collection.objects.get(pk=my_collection_pk)

        # Copy the nifti file into the proper location
        image = cls(description=my_desc, name=my_name, collection=my_collection)
        f = open(my_file)
        niftiFile = File(f)
        image.file.save(my_file_name, niftiFile)

        # If a .img file was loaded then load the correspoding .hdr file as well
        _, ext = os.path.splitext(my_file_name)
        print ext
        if ext in ['.img']:
            f = open(my_file[:-3] + "hdr")
            hdrFile = File(f)
            image.hdr_file.save(my_file_name[:-3] + "hdr", hdrFile)

        image.map_type = my_map_type

        # create JSON file for neurosynth viewer
        if os.path.exists(image.file.path):
            nifti_gz_file = ".".join(image.file.path.split(".")[:-1]) + '.nii.gz'
            nii = nb.load(image.file.path)
            nb.save(nii, nifti_gz_file)
            f = open(nifti_gz_file)
            image.nifti_gz_file.save(nifti_gz_file.split(os.path.sep)[-1], File(f), save=False)

        image.save()

        return image

    # Celery task to generate glass brain image on new/update
    def save(self):
        file_changed = False
        collection_changed = False
        if self.pk is not None:
            old_pk = Image.objects.get(pk=self.pk)
            if old_pk.file != self.file:
                file_changed = True
            if old_pk.collection != self.collection:
                collection_changed = True

        do_update = True if file_changed else False
        new_image = True if self.pk is None else False
        super(Image, self).save()

        if do_update or new_image:
            process_map.apply_async([self.pk])

        if self.subject_species == None and self.target_template_image:
            import neurovault.apps.statmaps.utils as nvutils
            self.subject_species = nvutils.infer_subject_species(self.target_template_image)

        if collection_changed:
            for field_name in self._meta.get_all_field_names():
                field_instance = getattr(self, field_name)
                if field_instance and isinstance(field_instance, FieldFile):
                    old_path = field_instance.path
                    new_name = upload_img_to(self, field_instance.name.split("/")[-1])
                    new_name = field_instance.storage.get_available_name(new_name)
                    new_path = field_instance.storage.path(new_name)
                    if not os.path.exists(os.path.dirname(new_path)):
                        os.mkdir(os.path.dirname(new_path))
                    shutil.copy(old_path, new_path)
                    field_instance.name = new_name
                    assert(old_path != new_path)
                    os.remove(old_path)
            super(Image, self).save()


    @classmethod
    def get_fixed_fields(cls):
        return super(Image, cls).get_fixed_fields() + ('target_template_image', 'age', 'gender',
                                                       'ethnicity', 'race', 'handedness',
                                                       'bis11_score', 'bis_bas_score',
                                                       'spsrq_score', 'BMI', 'fat_percentage',
                                                       'waist_hip_ratio', 'hours_since_last_meal',
                                                       'days_since_menstruation',
                                                       'mean_PDS_score', 'tanner_stage',) class Atlas(Image):
    label_description_file = models.FileField(
                                upload_to=upload_img_to,
                                null=False, blank=False,
                                storage=DoubleExtensionStorage(),
                                verbose_name='FSL compatible label description file (.xml)')

    class Meta:
        verbose_name_plural = "Atlases" class StatisticMap(BaseStatisticMap):
    fMRI_BOLD = 'fMRI-BOLD'
    fMRI_CBF = 'fMRI-CBF'
    fMRI_CBV = 'fMRI-CBV'
    Diffusion_MRI = 'Diffusion MRI'
    Structural_MRI = 'Structural MRI'
    PET_FDG = 'PET FDG'
    PET_15O = 'PET [15O]-water'
    PET_OTHER = 'PET other'
    MEG = 'MEG'
    EEG = 'EEG'
    Other = 'Other'
    MODALITY_CHOICES = (
        (fMRI_BOLD, 'fMRI-BOLD'),
        (fMRI_CBF, 'fMRI-CBF'),
        (fMRI_CBV, 'fMRI-CBV'),
        (Diffusion_MRI, 'Diffusion MRI'),
        (Structural_MRI, 'Structural MRI'),
        (PET_FDG, 'PET FDG'),
        (PET_15O, 'PET [15O]-water'),
        (PET_OTHER, 'PET other'),
        (MEG, 'MEG'),
        (EEG, 'EEG'),
        (Other, 'Other')
    )
    ignore_file_warning = models.BooleanField(blank=False, default=False, verbose_name='Ignore the warning',
                                              help_text="Ignore the warning when the map is sparse by nature, an ROI mask, or was acquired with limited field of view.")
    modality = models.CharField(verbose_name="Modality & Acquisition Type", help_text="Brain imaging procedure that was used to acquire the data.",
                                max_length=200, null=False, blank=False, choices=MODALITY_CHOICES)

    statistic_parameters = models.FloatField(help_text="Parameters of the null distribution of the test statistic, typically degrees of freedom (should be clear from the test statistic what these are).", null=True, verbose_name="Statistic parameters", blank=True)
    smoothness_fwhm = models.FloatField(help_text="Noise smoothness for statistical inference; this is the estimated smoothness used with Random Field Theory or a simulation-based inference method.", verbose_name="Smoothness FWHM", null=True, blank=True)
    contrast_definition = models.CharField(help_text="Exactly what terms are subtracted from what? Define these in terms of task or stimulus conditions (e.g., 'one-back task with objects versus zero-back task with objects') instead of underlying psychological concepts (e.g., 'working memory').", verbose_name="Contrast definition", max_length=200, null=True, blank=True)
    contrast_definition_cogatlas = models.CharField(help_text="Link to <a href='http://www.cognitiveatlas.org/'>Cognitive Atlas</a> definition of this contrast", verbose_name="Cognitive Atlas definition", max_length=200, null=True, blank=True)
    cognitive_paradigm_cogatlas = models.ForeignKey(CognitiveAtlasTask, help_text="Task (or lack of it) performed by the subjects in the scanner described using <a href='http://www.cognitiveatlas.org/' target='_blank'>Cognitive Atlas</a> terms",
                                                    verbose_name="Cognitive Atlas Paradigm", null=True, blank=False,
                                                    on_delete=models.PROTECT)
    cognitive_contrast_cogatlas = models.ForeignKey(CognitiveAtlasContrast, help_text="Link to <a href='http://www.cognitiveatlas.org/'>Cognitive Atlas</a> definition of this contrast",
                                                    verbose_name="Cognitive Atlas Contrast", null=True, blank=True,
                                                    on_delete=models.PROTECT)
    cognitive_paradigm_description_url = models.URLField(help_text="Link to a paper, poster, abstract or other form text describing in detail the task performed by the subject(s) in the scanner.",
                                                         verbose_name="Cognitive Paradigm Description URL",
                                                         null=True, blank=True)

    @classmethod
    def get_fixed_fields(cls):
        return super(StatisticMap, cls).get_fixed_fields() + (
            'modality', 'contrast_definition', 'cognitive_paradigm_cogatlas', 'cognitive_paradigm_description_url') class NIDMResults(BaseCollectionItem):
    ttl_file = models.FileField(upload_to=upload_nidm_to,
                    storage=NIDMStorage(),
                    null=True, blank=True,
                    verbose_name='Turtle serialization of NIDM Results (.ttl)')

    zip_file = models.FileField(upload_to=upload_nidm_to,
                    storage=NIDMStorage(),
                    null=False, blank=False, verbose_name='NIDM Results zip file')

    class Meta:
        verbose_name_plural = "NIDMResults"

    def get_absolute_url(self):
        return_args = [str(self.collection_id),self.name]
        url_name = 'view_nidm_results'
        if self.collection.private:
            return_args[0] = str(self.collection.private_token)
        return reverse(url_name, args=return_args)

    @staticmethod
    def get_form_class():
        from neurovault.apps.statmaps.forms import NIDMResultsForm
        return NIDMResultsForm class NIDMResultStatisticMap(BaseStatisticMap):
    nidm_results = models.ForeignKey(NIDMResults) class CognitiveAtlasTask(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, db_index=True)
    cog_atlas_id = models.CharField(primary_key=True, max_length=200, null=False, blank=False)

    def __str__(self):
        return self.name

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        cog_atlas_id = self.cog_atlas_id
        return reverse('view_task', args=[str(cog_atlas_id)])

    class Meta:
        ordering = ['name'] class CognitiveAtlasContrast(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False)
    cog_atlas_id = models.CharField(primary_key=True, max_length=200, null=False, blank=False)
    task = models.ForeignKey(CognitiveAtlasTask)

    def __str__(self):
        return self.name

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ['name'] class BaseStatisticMap(Image):
    T = 'T'
    Z = 'Z'
    F = 'F'
    X2 = 'X2'
    P = 'P'
    IP = 'IP'
    M ='M'
    U = 'U'
    R = 'R'
    Pa = 'Pa'
    OTHER = 'Other'
    S = 'S'
    G = 'G'
    M = 'M'
    A = 'A'
    V = 'V'
    MAP_TYPE_CHOICES = (
        (T, 'T map'),
        (Z, 'Z map'),
        (F, 'F map'),
        (X2, 'Chi squared map'),
        (P, 'P map (given null hypothesis)'),
        (IP, '1-P map ("inverted" probability)'),
        (M, 'multivariate-beta map'),
        (U, 'univariate-beta map'),
        (R, 'ROI/mask'),
        (Pa, 'parcellation'),
        (A, 'anatomical'),
        (V, 'variance'),
        (OTHER, 'other'),
    )
    ANALYSIS_LEVEL_CHOICES = (
        (S, 'single-subject'),
        (G, 'group'),
        (M, 'meta-analysis'),
        (OTHER, 'other'),
    )

    map_type = models.CharField(
                    help_text=("Type of statistic that is the basis of the inference"),
                    verbose_name="Map type",
                    max_length=200, null=False, blank=False, choices=MAP_TYPE_CHOICES)
    is_thresholded = models.NullBooleanField(null=True, blank=True)
    perc_bad_voxels = models.FloatField(null=True, blank=True)
    not_mni = models.NullBooleanField(null=True, blank=True)
    brain_coverage = models.FloatField(null=True, blank=True)
    perc_voxels_outside = models.FloatField(null=True, blank=True)
    analysis_level = models.CharField(
                    help_text=("What level of summary data was used as the input to this analysis?"),
                    verbose_name="Analysis level",
                    max_length=200, null=True, blank=False, choices=ANALYSIS_LEVEL_CHOICES)
    number_of_subjects = models.IntegerField(help_text="Number of subjects used to generate this map", null=True,
                                             verbose_name="No. of subjects", blank=False)

    def save(self):
        if self.perc_bad_voxels == None and self.file:
            import neurovault.apps.statmaps.utils as nvutils
            self.file.open()
            gzfileobj = GzipFile(filename=self.file.name, mode='rb', fileobj=self.file.file)
            nii = nb.Nifti1Image.from_file_map({'image': nb.FileHolder(self.file.name, gzfileobj)})
            self.is_thresholded, ratio_bad = nvutils.is_thresholded(nii)
            self.perc_bad_voxels = ratio_bad*100.0

        if self.brain_coverage == None and self.file:
            import neurovault.apps.statmaps.utils as nvutils
            self.file.open()
            gzfileobj = GzipFile(filename=self.file.name, mode='rb', fileobj=self.file.file)
            nii = nb.Nifti1Image.from_file_map({'image': nb.FileHolder(self.file.name, gzfileobj)})
            self.not_mni, self.brain_coverage, self.perc_voxels_outside = nvutils.not_in_mni(nii, self.target_template_image)

        if self.map_type == self.OTHER:
            import neurovault.apps.statmaps.utils as nvutils
            self.file.open()
            gzfileobj = GzipFile(filename=self.file.name, mode='rb', fileobj=self.file.file)
            nii = nb.Nifti1Image.from_file_map({'image': nb.FileHolder(self.file.name, gzfileobj)})
            self.map_type = nvutils.infer_map_type(nii)

        # Calculation of image reduced_representation and comparisons
        file_changed = False
        if self.pk is not None:
            existing = Image.objects.get(pk=self.pk)
            if existing.file != self.file:
                file_changed = True
        do_update = True if file_changed else False
        new_image = True if self.pk is None else False

        # If we have an update, delete old pkl and comparisons first before saving
        if do_update and self.collection:
            if self.reduced_representation: # not applicable for private collections
                self.reduced_representation.delete()

                # If more than one metric is added to NeuroVault, this must also filter based on metric
                comparisons = Comparison.objects.filter(Q(image1=self) | Q(image2=self))
                if comparisons:
                    comparisons.delete()
        super(BaseStatisticMap, self).save()

        # Calculate comparisons
        if do_update or new_image:
            run_voxelwise_pearson_similarity.apply_async([self.pk])

        self.file.close()

    @classmethod
    def get_fixed_fields(cls):
        return super(BaseStatisticMap, cls).get_fixed_fields() + (
            'map_type', 'analysis_level', 'number_of_subjects')

    class Meta:
        abstract = True DEFAULT_TEMPLATE = 'GenericMNI' class BaseCollectionItem(PolymorphicModel, models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, db_index=True)
    description = models.TextField(blank=True)
    collection = models.ForeignKey(Collection)
    add_date = models.DateTimeField('date published', auto_now_add=True)
    modify_date = models.DateTimeField('date modified', auto_now=True)
    tags = TaggableManager(through=ValueTaggedItem, blank=True)
    is_valid = models.BooleanField(default=True)

    def __unicode__(self):
        return self.name

    def set_name(self, new_name):
        self.name = new_name

    def save(self):
        self.collection.modify_date = datetime.now()
        self.collection.save()
        super(BaseCollectionItem, self).save()
        

    def delete(self):
        self.collection.modify_date = datetime.now()
        self.collection.save()
        super(BaseCollectionItem, self).delete()

    @classmethod
    def get_fixed_fields(cls):
        return ('name', 'description', 'figure') class Metaanalysis(models.Model):
    owner = models.ForeignKey(User)
    name = models.CharField(max_length=200, unique=True, null=False)
    description = models.TextField(blank=True, null=True)
    maps = models.ManyToManyField(StatisticMap, blank=True, null=True)
    status =  models.CharField(choices=[('active', 'active'), ('inactive', 'inactive'),
                                        ('completed', 'completed')],
                               max_length=200, blank=True, null=True, default='active')
    output_maps = models.ForeignKey(Collection, blank=True, null=True)

    def save(self, *args, **kwargs):
        if self.status == 'active':
            for m in Metaanalysis.objects.filter(owner=self.owner).filter(status='active'):
                if m.pk != self.pk :
                    m.status = 'inactive'
                    m.save()
        super(Metaanalysis, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('view_metaanalysis', args=[str(self.id)])
import csv
import functools
import gzip
import json
import nibabel as nib
import shutil
import numpy as np
import os
import re
import shutil
import tarfile
import tempfile
import traceback
import urllib
import zipfile
import zipstream
import neurovault
    import numpy as np
from collections import OrderedDict
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
from django.core.files.base import ContentFile
from django.db.models import Q
from django.db.models.aggregates import Count
from django.http import Http404, HttpResponse, StreamingHttpResponse
from django.http.response import HttpResponseRedirect, HttpResponseForbidden
from django.shortcuts import get_object_or_404, render_to_response, render, redirect
from django.template.context import RequestContext
from django.utils.encoding import filepath_to_uri
from django.views.decorators.csrf import csrf_exempt
from django_datatables_view.base_datatable_view import BaseDatatableView
from django.core.urlresolvers import reverse
from fnmatch import fnmatch
from guardian.shortcuts import get_objects_for_user
from nidmviewer.viewer import generate
from pybraincompare.compare.scatterplot import scatterplot_compare_vector
from nidmresults.graph import Graph
from rest_framework.renderers import JSONRenderer
from sendfile import sendfile
from sklearn.externals import joblib
from xml.dom import minidom
from django.core.files.uploadedfile import SimpleUploadedFile
from nimare.meta.ibma import stouffers, weighted_stouffers
from nimare.utils import t_to_z
from nilearn.masking import apply_mask
from nilearn.image import resample_to_img
from neurovault.apps.statmaps.models import get_possible_templates, DEFAULT_TEMPLATE
from neurovault import settings
from neurovault.apps.statmaps.ahba import calculate_gene_expression_similarity
from neurovault.apps.statmaps.forms import CollectionForm, UploadFileForm, SimplifiedStatisticMapForm,NeuropowerStatisticMapForm,\
    StatisticMapForm, EditStatisticMapForm, OwnerCollectionForm, EditAtlasForm, AtlasForm, \
    EditNIDMResultStatisticMapForm, NIDMResultsForm, NIDMViewForm, AddStatisticMapForm, MetaanalysisForm
from neurovault.apps.statmaps.models import Collection, Image, Atlas, \
    StatisticMap, NIDMResults, NIDMResultStatisticMap, \
    CognitiveAtlasTask, CognitiveAtlasContrast, BaseStatisticMap, \
    DEFAULT_TEMPLATE, BaseCollectionItem, Metaanalysis
from neurovault.apps.statmaps.tasks import save_resampled_transformation_single
from neurovault.apps.statmaps.utils import split_filename, generate_pycortex_volume, \
    generate_pycortex_static, generate_url_token, HttpRedirectException, get_paper_properties, \
    get_file_ctime, detect_4D, split_4D_to_3D, splitext_nii_gz, mkdir_p, \
    send_email_notification, populate_nidm_results, get_server_url, populate_feat_directory, \
    detect_feat_directory, format_image_collection_names, is_search_compatible, \
    get_similar_images
from neurovault.apps.statmaps.utils import is_target_template_image_pycortex_compatible, is_target_template_image_neurosynth_compatible
from neurovault.apps.statmaps.voxel_query_functions import *
from . import image_metadata
    from cogat_functions import get_task_graph
        result = stouffers(z_data, mask_img, inference='ffx', null='theoretical', corr='FWE',
                           two_sided=True)

    z_map = StatisticMap(name='FWE corrected Z map', description='', collection=new_collection,
                         modality='Other',
                         map_type='Z',
                         analysis_level='M',
                         target_template_image='GenericMNI')
    p_map = StatisticMap(name='FWE corrected P map', description='', collection=new_collection,
                         modality='Other',
                         map_type='P',
                         analysis_level='M',
                         number_of_subjects=-1,
                         target_template_image='GenericMNI')
    if do_weighted:
        ffx_map = StatisticMap(name='FFX map', description='', collection=new_collection,
                               modality='Other',
                               map_type='Other',
                               analysis_level='M',
                               target_template_image='GenericMNI')

    tmp_dir = tempfile.mkdtemp()
    try:
        z_path = os.path.join(tmp_dir, 'z_fwe_corr.nii.gz')
        result.images['z'].to_filename(z_path)
        z_map.file = SimpleUploadedFile('z_fwe_corr.nii.gz',
                                        open(z_path).read())
        z_map.save()

        p_path = os.path.join(tmp_dir, 'p_fwe_corr.nii.gz')
        result.images['p'].to_filename(p_path)
        p_map.file = SimpleUploadedFile('p_fwe_corr.nii.gz',
                                        open(p_path).read())
        p_map.save()
        if do_weighted:
            ffx_path = os.path.join(tmp_dir, 'ffx_stat.nii.gz')
            result.images['ffx_stat'].to_filename(ffx_path)
            ffx_map.file = SimpleUploadedFile('ffx_stat.nii.gz',
                                            open(ffx_path).read())
            ffx_map.save()
    finally:
        shutil.rmtree(tmp_dir)

    metaanalysis.output_maps = new_collection
    metaanalysis.status='completed'
    metaanalysis.save()

    return HttpResponseRedirect(new_collection.get_absolute_url())

@login_required
def toggle_active_metaanalysis(request, pk, collection_cid=None):
    image = get_image(pk,collection_cid,request)
    metaanalysis = Metaanalysis.objects.filter(owner=request.user).filter(status='active')[0]
    if request.user != metaanalysis.owner:
        return HttpResponseForbidden()
    if metaanalysis in image.metaanalysis_set.all():
        metaanalysis.maps.remove(image)
        new_status = 'out'
    else:
        metaanalysis.maps.add(image)
        new_status = 'in'
    return JSONResponse({'new_status': new_status})

@login_required
def activate_metaanalysis(request, metaanalysis_id):
    metaanalysis = Metaanalysis.objects.get(pk=metaanalysis_id)
    if request.user != metaanalysis.owner or metaanalysis.status == 'completed':
        return HttpResponseForbidden()
    metaanalysis.status = 'active'
    metaanalysis.save()
    return HttpResponseRedirect(metaanalysis.get_absolute_url())


@login_required
def edit_collection(request, cid=None):
    '''edit_collection is a view to edit a collection, meaning showing the edit form for an existing collection, creating a new collection, or passing POST data to update an existing collection
    :param cid: statmaps.models.Collection.pk the primary key of the collection. If none, will generate form to make a new collection.
    '''
    page_header = "Add new collection"
    if cid:
        collection = get_collection(cid, request)
        edit_permission = request.user.has_perm('statmaps.change_collection', collection)
        is_owner = request.user.has_perm('statmaps.delete_collection', collection)
        page_header = 'Edit collection'
        if not edit_permission:
            return HttpResponseForbidden()
    else:
        is_owner = True
        collection = Collection(owner=request.user)
    if request.method == "POST":
        if is_owner:
            form = OwnerCollectionForm(request.POST, request.FILES, instance=collection)
        else:
            form = CollectionForm(request.POST, request.FILES, instance=collection)
        if form.is_valid():
            previous_contribs = set()
            if form.instance.pk is not None:
                previous_contribs = set(form.instance.contributors.all())
            collection = form.save(commit=False)
            if collection.private and collection.private_token is None:
                collection.private_token = generate_url_token()
            collection.save()

            if is_owner:
                form.save_m2m()  # save contributors
                current_contribs = set(collection.contributors.all())
                new_contribs = list(current_contribs.difference(previous_contribs))
                context = {
                    'owner': collection.owner.username,
                    'collection': collection.name,
                    'url': get_server_url(request) + collection.get_absolute_url(),
                }
                subj = '%s has added you to a NeuroVault collection' % context['owner']
                send_email_notification('new_contributor', subj, new_contribs, context)

            return HttpResponseRedirect(collection.get_absolute_url())
    else:
        if is_owner:
            form = OwnerCollectionForm(instance=collection)
        else:

            form = CollectionForm(instance=collection)