{"action":"created","comment":{"url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/comments/57923994","id":57923994,"diff_hunk":"@@ -0,0 +1,284 @@\n+package org.transmartproject.batch.startup\n+\n+import com.google.common.base.Function\n+import com.google.common.collect.Maps\n+import groovy.transform.TypeChecked\n+import groovy.transform.TypeCheckingMode\n+import groovy.util.logging.Slf4j\n+import org.springframework.batch.core.JobParameter\n+import org.springframework.batch.core.JobParameters\n+import org.transmartproject.batch.backout.BackoutJobSpecification\n+import org.transmartproject.batch.clinical.ClinicalJobSpecification\n+import org.transmartproject.batch.gwas.GwasJobSpecification\n+import org.transmartproject.batch.highdim.acgh.data.AcghDataJobSpecification\n+import org.transmartproject.batch.highdim.acgh.platform.AcghAnnotationJobSpecification\n+import org.transmartproject.batch.highdim.metabolomics.data.MetabolomicsDataJobSpecification\n+import org.transmartproject.batch.highdim.metabolomics.platform.MetabolomicsAnnotationJobSpecification\n+import org.transmartproject.batch.highdim.mrna.data.MrnaDataJobSpecification\n+import org.transmartproject.batch.highdim.mrna.platform.MrnaAnnotationJobSpecification\n+import org.transmartproject.batch.highdim.proteomics.data.ProteomicsDataJobSpecification\n+import org.transmartproject.batch.highdim.proteomics.platform.ProteomicsAnnotationJobSpecification\n+import org.transmartproject.batch.highdim.rnaseq.data.RnaSeqDataJobSpecification\n+import org.transmartproject.batch.highdim.rnaseq.platform.RnaSeqAnnotationJobSpecification\n+import org.transmartproject.batch.i2b2.I2b2JobSpecification\n+import org.transmartproject.batch.support.StringUtils\n+import org.transmartproject.batch.tag.TagsLoadJobSpecification\n+\n+import java.nio.file.Files\n+import java.nio.file.Path\n+\n+/**\n+ * Gets all details needed for running a job.\n+ */\n+@TypeChecked\n+@Slf4j\n+final class JobDetails {\n+\n+    private final static Map<String, Class<? extends JobSpecification>> DATA_TYPE_TO_JOB_SPEC = [\n+            'clinical'               : ClinicalJobSpecification,\n+            'annotation'             : MrnaAnnotationJobSpecification,\n+            'tags'                   : TagsLoadJobSpecification,\n+            'expression'             : MrnaDataJobSpecification,\n+            'metabolomics_annotation': MetabolomicsAnnotationJobSpecification,\n+            'metabolomics'           : MetabolomicsDataJobSpecification,\n+            'i2b2'                   : I2b2JobSpecification,\n+            'proteomics_annotation'  : ProteomicsAnnotationJobSpecification,\n+            'proteomics'             : ProteomicsDataJobSpecification,\n+            'gwas'                   : GwasJobSpecification,\n+            'backout'                : BackoutJobSpecification,\n+            'rnaseq_annotation'      : RnaSeqAnnotationJobSpecification,\n+            'rnaseq'                 : RnaSeqDataJobSpecification,\n+            'acgh_annotation'        : AcghAnnotationJobSpecification,\n+            'acgh'                   : AcghDataJobSpecification,\n+    ]\n+    public static final String STUDY_PARAMS_FILE_NAME = 'study' + PARAMS_FILE_EXTENSION\n+    public static final String PARAMS_FILE_EXTENSION = '.params'\n+\n+    private final Map<String, String> params = Maps.newHashMap()\n+\n+    private Path dataTypeParamsFilePath\n+\n+    private String typeName\n+\n+    private List<? extends ExternalJobParametersModule> modules\n+\n+    private Set<String> fileParameterKeys\n+\n+    Class<?> jobPath\n+\n+    private JobDetails() {}\n+\n+    /**\n+     * Detects the job to run by '*.params' file.\n+     * @param filePath - a *.params file\n+     * @param overrides - parameters to override ones in the *.params file\n+     * @return The job details.\n+     */\n+    static JobDetails fromFile(Path filePath, Map<String, String> overrides = [:]) {\n+\n+        String dataType = getDataTypeByProperyFile(filePath)\n+        JobSpecification spec = getJobSpecificationByDataType(dataType)\n+\n+        JobDetails instance = new JobDetails()\n+        instance.dataTypeParamsFilePath = filePath\n+        instance.typeName = dataType\n+        instance.modules = spec.jobParametersModules\n+        instance.fileParameterKeys = getFileParameterKeys(spec)\n+        instance.jobPath = spec.jobPath\n+\n+        Map<String, String> dataTypeParams = parseContent(filePath)\n+        if (isStudyRelatedSpecification(spec)) {\n+            Path studyParamsFilePath = findStudyFilePath(filePath)\n+            if (studyParamsFilePath) {\n+                Map<String, String> studyParams = parseContent(studyParamsFilePath)\n+                dataTypeParams.each { item ->\n+                    if (studyParams.containsKey(item.key)) {\n+                        log.warn(\"Study-wise ${item.key} parameter gets overriden by data type one:\" +\n+                                \" ${studyParams[item.value]} -> ${item.value}\")\n+                    }\n+                }\n+                instance.params << studyParams\n+            } else {\n+                log.warn('No study parameters file is found.' +\n+                        ' Rely on data type parameters file for specifying study parameters.')","path":"src/main/groovy/org/transmartproject/batch/startup/JobDetails.groovy","position":103,"original_position":103,"commit_id":"445395c15e702f996128902a9975bde90cde6297","original_commit_id":"445395c15e702f996128902a9975bde90cde6297","user":{"login":"cataphract","id":496394,"avatar_url":"https://avatars.githubusercontent.com/u/496394?v=3","gravatar_id":"","url":"https://api.github.com/users/cataphract","html_url":"https://github.com/cataphract","followers_url":"https://api.github.com/users/cataphract/followers","following_url":"https://api.github.com/users/cataphract/following{/other_user}","gists_url":"https://api.github.com/users/cataphract/gists{/gist_id}","starred_url":"https://api.github.com/users/cataphract/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cataphract/subscriptions","organizations_url":"https://api.github.com/users/cataphract/orgs","repos_url":"https://api.github.com/users/cataphract/repos","events_url":"https://api.github.com/users/cataphract/events{/privacy}","received_events_url":"https://api.github.com/users/cataphract/received_events","type":"User","site_admin":false},"body":"I don't think this should trigger a warning. I see no reason you should have two params files if you only have clinical data. Plus, I think this will trigger warnings for all test cases.","created_at":"2016-03-30T16:57:01Z","updated_at":"2016-03-30T16:57:01Z","html_url":"https://github.com/thehyve/transmart-batch/pull/66#discussion_r57923994","pull_request_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66","_links":{"self":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/comments/57923994"},"html":{"href":"https://github.com/thehyve/transmart-batch/pull/66#discussion_r57923994"},"pull_request":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66"}}},"pull_request":{"url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66","id":64651439,"html_url":"https://github.com/thehyve/transmart-batch/pull/66","diff_url":"https://github.com/thehyve/transmart-batch/pull/66.diff","patch_url":"https://github.com/thehyve/transmart-batch/pull/66.patch","issue_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/66","number":66,"state":"open","locked":false,"title":"New way to place input files (old way supported).","user":{"login":"forus","id":492175,"avatar_url":"https://avatars.githubusercontent.com/u/492175?v=3","gravatar_id":"","url":"https://api.github.com/users/forus","html_url":"https://github.com/forus","followers_url":"https://api.github.com/users/forus/followers","following_url":"https://api.github.com/users/forus/following{/other_user}","gists_url":"https://api.github.com/users/forus/gists{/gist_id}","starred_url":"https://api.github.com/users/forus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/forus/subscriptions","organizations_url":"https://api.github.com/users/forus/orgs","repos_url":"https://api.github.com/users/forus/repos","events_url":"https://api.github.com/users/forus/events{/privacy}","received_events_url":"https://api.github.com/users/forus/received_events","type":"User","site_admin":false},"body":"- Introduced study.params\r\n- <data type>.params file does not have to have <data type> folder next to it.\r\n- No guessing of STUDY_ID from the folder name.","created_at":"2016-03-30T15:53:23Z","updated_at":"2016-03-30T16:57:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c24aa853a3566c2518cf598ca148ba1b1e5e7315","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66/commits","review_comments_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66/comments","review_comment_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/66/comments","statuses_url":"https://api.github.com/repos/thehyve/transmart-batch/statuses/445395c15e702f996128902a9975bde90cde6297","head":{"label":"thehyve:multiple-data-sets","ref":"multiple-data-sets","sha":"445395c15e702f996128902a9975bde90cde6297","user":{"login":"thehyve","id":1502440,"avatar_url":"https://avatars.githubusercontent.com/u/1502440?v=3","gravatar_id":"","url":"https://api.github.com/users/thehyve","html_url":"https://github.com/thehyve","followers_url":"https://api.github.com/users/thehyve/followers","following_url":"https://api.github.com/users/thehyve/following{/other_user}","gists_url":"https://api.github.com/users/thehyve/gists{/gist_id}","starred_url":"https://api.github.com/users/thehyve/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thehyve/subscriptions","organizations_url":"https://api.github.com/users/thehyve/orgs","repos_url":"https://api.github.com/users/thehyve/repos","events_url":"https://api.github.com/users/thehyve/events{/privacy}","received_events_url":"https://api.github.com/users/thehyve/received_events","type":"Organization","site_admin":false},"repo":{"id":23545751,"name":"transmart-batch","full_name":"thehyve/transmart-batch","owner":{"login":"thehyve","id":1502440,"avatar_url":"https://avatars.githubusercontent.com/u/1502440?v=3","gravatar_id":"","url":"https://api.github.com/users/thehyve","html_url":"https://github.com/thehyve","followers_url":"https://api.github.com/users/thehyve/followers","following_url":"https://api.github.com/users/thehyve/following{/other_user}","gists_url":"https://api.github.com/users/thehyve/gists{/gist_id}","starred_url":"https://api.github.com/users/thehyve/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thehyve/subscriptions","organizations_url":"https://api.github.com/users/thehyve/orgs","repos_url":"https://api.github.com/users/thehyve/repos","events_url":"https://api.github.com/users/thehyve/events{/privacy}","received_events_url":"https://api.github.com/users/thehyve/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/thehyve/transmart-batch","description":"tranSMART pipeline alternative to ETL, using Spring Batch","fork":false,"url":"https://api.github.com/repos/thehyve/transmart-batch","forks_url":"https://api.github.com/repos/thehyve/transmart-batch/forks","keys_url":"https://api.github.com/repos/thehyve/transmart-batch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/thehyve/transmart-batch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/thehyve/transmart-batch/teams","hooks_url":"https://api.github.com/repos/thehyve/transmart-batch/hooks","issue_events_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/events{/number}","events_url":"https://api.github.com/repos/thehyve/transmart-batch/events","assignees_url":"https://api.github.com/repos/thehyve/transmart-batch/assignees{/user}","branches_url":"https://api.github.com/repos/thehyve/transmart-batch/branches{/branch}","tags_url":"https://api.github.com/repos/thehyve/transmart-batch/tags","blobs_url":"https://api.github.com/repos/thehyve/transmart-batch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/thehyve/transmart-batch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/thehyve/transmart-batch/git/refs{/sha}","trees_url":"https://api.github.com/repos/thehyve/transmart-batch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/thehyve/transmart-batch/statuses/{sha}","languages_url":"https://api.github.com/repos/thehyve/transmart-batch/languages","stargazers_url":"https://api.github.com/repos/thehyve/transmart-batch/stargazers","contributors_url":"https://api.github.com/repos/thehyve/transmart-batch/contributors","subscribers_url":"https://api.github.com/repos/thehyve/transmart-batch/subscribers","subscription_url":"https://api.github.com/repos/thehyve/transmart-batch/subscription","commits_url":"https://api.github.com/repos/thehyve/transmart-batch/commits{/sha}","git_commits_url":"https://api.github.com/repos/thehyve/transmart-batch/git/commits{/sha}","comments_url":"https://api.github.com/repos/thehyve/transmart-batch/comments{/number}","issue_comment_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/comments{/number}","contents_url":"https://api.github.com/repos/thehyve/transmart-batch/contents/{+path}","compare_url":"https://api.github.com/repos/thehyve/transmart-batch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/thehyve/transmart-batch/merges","archive_url":"https://api.github.com/repos/thehyve/transmart-batch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/thehyve/transmart-batch/downloads","issues_url":"https://api.github.com/repos/thehyve/transmart-batch/issues{/number}","pulls_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls{/number}","milestones_url":"https://api.github.com/repos/thehyve/transmart-batch/milestones{/number}","notifications_url":"https://api.github.com/repos/thehyve/transmart-batch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/thehyve/transmart-batch/labels{/name}","releases_url":"https://api.github.com/repos/thehyve/transmart-batch/releases{/id}","deployments_url":"https://api.github.com/repos/thehyve/transmart-batch/deployments","created_at":"2014-09-01T14:46:36Z","updated_at":"2016-03-18T10:38:27Z","pushed_at":"2016-03-30T16:32:12Z","git_url":"git://github.com/thehyve/transmart-batch.git","ssh_url":"git@github.com:thehyve/transmart-batch.git","clone_url":"https://github.com/thehyve/transmart-batch.git","svn_url":"https://github.com/thehyve/transmart-batch","homepage":null,"size":1358,"stargazers_count":4,"watchers_count":4,"language":"Groovy","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":3,"forks":7,"open_issues":3,"watchers":4,"default_branch":"master"}},"base":{"label":"thehyve:master","ref":"master","sha":"229f8ccc2ceeeacfe783aafa666999482b44556b","user":{"login":"thehyve","id":1502440,"avatar_url":"https://avatars.githubusercontent.com/u/1502440?v=3","gravatar_id":"","url":"https://api.github.com/users/thehyve","html_url":"https://github.com/thehyve","followers_url":"https://api.github.com/users/thehyve/followers","following_url":"https://api.github.com/users/thehyve/following{/other_user}","gists_url":"https://api.github.com/users/thehyve/gists{/gist_id}","starred_url":"https://api.github.com/users/thehyve/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thehyve/subscriptions","organizations_url":"https://api.github.com/users/thehyve/orgs","repos_url":"https://api.github.com/users/thehyve/repos","events_url":"https://api.github.com/users/thehyve/events{/privacy}","received_events_url":"https://api.github.com/users/thehyve/received_events","type":"Organization","site_admin":false},"repo":{"id":23545751,"name":"transmart-batch","full_name":"thehyve/transmart-batch","owner":{"login":"thehyve","id":1502440,"avatar_url":"https://avatars.githubusercontent.com/u/1502440?v=3","gravatar_id":"","url":"https://api.github.com/users/thehyve","html_url":"https://github.com/thehyve","followers_url":"https://api.github.com/users/thehyve/followers","following_url":"https://api.github.com/users/thehyve/following{/other_user}","gists_url":"https://api.github.com/users/thehyve/gists{/gist_id}","starred_url":"https://api.github.com/users/thehyve/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thehyve/subscriptions","organizations_url":"https://api.github.com/users/thehyve/orgs","repos_url":"https://api.github.com/users/thehyve/repos","events_url":"https://api.github.com/users/thehyve/events{/privacy}","received_events_url":"https://api.github.com/users/thehyve/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/thehyve/transmart-batch","description":"tranSMART pipeline alternative to ETL, using Spring Batch","fork":false,"url":"https://api.github.com/repos/thehyve/transmart-batch","forks_url":"https://api.github.com/repos/thehyve/transmart-batch/forks","keys_url":"https://api.github.com/repos/thehyve/transmart-batch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/thehyve/transmart-batch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/thehyve/transmart-batch/teams","hooks_url":"https://api.github.com/repos/thehyve/transmart-batch/hooks","issue_events_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/events{/number}","events_url":"https://api.github.com/repos/thehyve/transmart-batch/events","assignees_url":"https://api.github.com/repos/thehyve/transmart-batch/assignees{/user}","branches_url":"https://api.github.com/repos/thehyve/transmart-batch/branches{/branch}","tags_url":"https://api.github.com/repos/thehyve/transmart-batch/tags","blobs_url":"https://api.github.com/repos/thehyve/transmart-batch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/thehyve/transmart-batch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/thehyve/transmart-batch/git/refs{/sha}","trees_url":"https://api.github.com/repos/thehyve/transmart-batch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/thehyve/transmart-batch/statuses/{sha}","languages_url":"https://api.github.com/repos/thehyve/transmart-batch/languages","stargazers_url":"https://api.github.com/repos/thehyve/transmart-batch/stargazers","contributors_url":"https://api.github.com/repos/thehyve/transmart-batch/contributors","subscribers_url":"https://api.github.com/repos/thehyve/transmart-batch/subscribers","subscription_url":"https://api.github.com/repos/thehyve/transmart-batch/subscription","commits_url":"https://api.github.com/repos/thehyve/transmart-batch/commits{/sha}","git_commits_url":"https://api.github.com/repos/thehyve/transmart-batch/git/commits{/sha}","comments_url":"https://api.github.com/repos/thehyve/transmart-batch/comments{/number}","issue_comment_url":"https://api.github.com/repos/thehyve/transmart-batch/issues/comments{/number}","contents_url":"https://api.github.com/repos/thehyve/transmart-batch/contents/{+path}","compare_url":"https://api.github.com/repos/thehyve/transmart-batch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/thehyve/transmart-batch/merges","archive_url":"https://api.github.com/repos/thehyve/transmart-batch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/thehyve/transmart-batch/downloads","issues_url":"https://api.github.com/repos/thehyve/transmart-batch/issues{/number}","pulls_url":"https://api.github.com/repos/thehyve/transmart-batch/pulls{/number}","milestones_url":"https://api.github.com/repos/thehyve/transmart-batch/milestones{/number}","notifications_url":"https://api.github.com/repos/thehyve/transmart-batch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/thehyve/transmart-batch/labels{/name}","releases_url":"https://api.github.com/repos/thehyve/transmart-batch/releases{/id}","deployments_url":"https://api.github.com/repos/thehyve/transmart-batch/deployments","created_at":"2014-09-01T14:46:36Z","updated_at":"2016-03-18T10:38:27Z","pushed_at":"2016-03-30T16:32:12Z","git_url":"git://github.com/thehyve/transmart-batch.git","ssh_url":"git@github.com:thehyve/transmart-batch.git","clone_url":"https://github.com/thehyve/transmart-batch.git","svn_url":"https://github.com/thehyve/transmart-batch","homepage":null,"size":1358,"stargazers_count":4,"watchers_count":4,"language":"Groovy","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":3,"forks":7,"open_issues":3,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66"},"html":{"href":"https://github.com/thehyve/transmart-batch/pull/66"},"issue":{"href":"https://api.github.com/repos/thehyve/transmart-batch/issues/66"},"comments":{"href":"https://api.github.com/repos/thehyve/transmart-batch/issues/66/comments"},"review_comments":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66/comments"},"review_comment":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/thehyve/transmart-batch/pulls/66/commits"},"statuses":{"href":"https://api.github.com/repos/thehyve/transmart-batch/statuses/445395c15e702f996128902a9975bde90cde6297"}}}}