{"action":"created","comment":{"url":"https://api.github.com/repos/dmwm/WMCore/pulls/comments/71207321","id":71207321,"diff_hunk":"@@ -954,44 +925,44 @@ def customizePerJob(self, job):\n         if job.get('proxyPath'):\n             jdl.append('x509userproxy = %s\\n' % job['proxyPath'])\n \n-        if job.get('requestName'):\n-            jdl.append('+WMAgent_RequestName = \"%s\"\\n' % job['requestName'])\n-            m = GROUP_NAME_RE.match(job['requestName'])\n-            if m:\n-                jdl.append('+CMSGroups = %s\\n' % classad.quote(m.groups()[0]))\n-\n-        if job.get('taskName'):\n-            jdl.append('+WMAgent_SubTaskName = \"%s\"\\n' % job['taskName'])\n+        jdl.append('+WMAgent_RequestName = \"%s\"\\n' % job['requestName'])\n+        match = GROUP_NAME_RE.match(job['requestName'])\n+        if match:\n+            jdl.append('+CMSGroups = %s\\n' % classad.quote(match.groups()[0]))\n+        else:\n+            jdl.append('+CMSGroups = undefined\\n')\n \n-        if job.get('taskType'):\n-            jdl.append('+CMS_JobType = \"%s\"\\n' % job['taskType'])\n+        jdl.append('+WMAgent_SubTaskName = \"%s\"\\n' % job['taskName'])\n+        jdl.append('+CMS_JobType = \"%s\"\\n' % job['taskType'])\n \n         # Handling for AWS, cloud and opportunistic resources\n         jdl.append('+AllowOpportunistic = %s\\n' % job.get('allowOpportunistic', False))\n \n         # dataset info\n         if job.get('inputDataset'):\n             jdl.append('+DESIRED_CMSDataset = \"%s\"\\n' % job['inputDataset'])\n+        else:\n+            jdl.append('+DESIRED_CMSDataset = undefined\\n')\n         if job.get('inputDatasetLocations'):\n             jdl.append('+DESIRED_CMSDataLocations = \"%s\"\\n' % ','.join(job['inputDatasetLocations']))\n+        else:\n+            jdl.append('+DESIRED_CMSDataLocations = undefined\\n')\n \n         # HighIO jobs\n-        highIOjob = job.get('highIOjob')\n-        if highIOjob:\n-            jdl.append('+Requestioslots = 1\\n')\n-\n-        # Performance estimates\n-        if job.get('estimatedJobTime'):\n-            jdl.append('+MaxWallTimeMins = %d\\n' % int(job['estimatedJobTime'] / 60.0))\n-        if job.get('estimatedMemoryUsage'):\n-            jdl.append('request_memory = %d\\n' % int(job['estimatedMemoryUsage']))\n-        if job.get('estimatedDiskUsage'):\n-            jdl.append('request_disk = %d\\n' % int(job['estimatedDiskUsage']))\n-\n-        # Set up JDL for multithreaded jobs\n-        if job.get('numberOfCores', 1) > 1:\n-            jdl.append('machine_count = 1\\n')\n-            jdl.append('request_cpus = %s\\n' % job.get('numberOfCores', 1))\n+        jdl.append('+Requestioslots = %d\\n' % job.get('highIOjob', 0))\n+\n+        # Performance and resource estimates\n+        numberOfCores = job.get('numberOfCores', 1)\n+        requestMemory = int(job['estimatedMemoryUsage']) if job.get('estimatedMemoryUsage', None) else 1000\n+        requestDisk = int(job['estimatedDiskUsage']) if job.get('estimatedDiskUsage', None) else 20*1000*1000*numberOfCores\n+        maxWallTimeMins = int(job['estimatedJobTime'])/60.0 if job.get('estimatedJobTime', None) else 12*60","path":"src/python/WMCore/BossAir/Plugins/PyCondorPlugin.py","position":262,"original_position":262,"commit_id":"5a142510117d43e7f66fd723d142bf82b8c507c0","original_commit_id":"5a142510117d43e7f66fd723d142bf82b8c507c0","user":{"login":"amaltaro","id":1798520,"avatar_url":"https://avatars.githubusercontent.com/u/1798520?v=3","gravatar_id":"","url":"https://api.github.com/users/amaltaro","html_url":"https://github.com/amaltaro","followers_url":"https://api.github.com/users/amaltaro/followers","following_url":"https://api.github.com/users/amaltaro/following{/other_user}","gists_url":"https://api.github.com/users/amaltaro/gists{/gist_id}","starred_url":"https://api.github.com/users/amaltaro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amaltaro/subscriptions","organizations_url":"https://api.github.com/users/amaltaro/orgs","repos_url":"https://api.github.com/users/amaltaro/repos","events_url":"https://api.github.com/users/amaltaro/events{/privacy}","received_events_url":"https://api.github.com/users/amaltaro/received_events","type":"User","site_admin":false},"body":"`requestDisk` and `maxWallTimeMins` are not reasonable at all. \r\nI think all the most important task types would have these estimates, leaving these default numbers only for logcollect, cleanup, etc. How about we set the default WT to 4h (maybe 8h then) and the disk to 5GB (10GB if you prefer).","created_at":"2016-07-18T18:59:13Z","updated_at":"2016-07-18T18:59:13Z","html_url":"https://github.com/dmwm/WMCore/pull/6925#discussion_r71207321","pull_request_url":"https://api.github.com/repos/dmwm/WMCore/pulls/6925","_links":{"self":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/comments/71207321"},"html":{"href":"https://github.com/dmwm/WMCore/pull/6925#discussion_r71207321"},"pull_request":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/6925"}}},"pull_request":{"url":"https://api.github.com/repos/dmwm/WMCore/pulls/6925","id":73424456,"html_url":"https://github.com/dmwm/WMCore/pull/6925","diff_url":"https://github.com/dmwm/WMCore/pull/6925.diff","patch_url":"https://github.com/dmwm/WMCore/pull/6925.patch","issue_url":"https://api.github.com/repos/dmwm/WMCore/issues/6925","number":6925,"state":"open","locked":false,"title":"make more efficient use of condor bulk submission","user":{"login":"hufnagel","id":1798355,"avatar_url":"https://avatars.githubusercontent.com/u/1798355?v=3","gravatar_id":"","url":"https://api.github.com/users/hufnagel","html_url":"https://github.com/hufnagel","followers_url":"https://api.github.com/users/hufnagel/followers","following_url":"https://api.github.com/users/hufnagel/following{/other_user}","gists_url":"https://api.github.com/users/hufnagel/gists{/gist_id}","starred_url":"https://api.github.com/users/hufnagel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hufnagel/subscriptions","organizations_url":"https://api.github.com/users/hufnagel/orgs","repos_url":"https://api.github.com/users/hufnagel/repos","events_url":"https://api.github.com/users/hufnagel/events{/privacy}","received_events_url":"https://api.github.com/users/hufnagel/received_events","type":"User","site_admin":false},"body":"Ok, it seems the only reason we needed to pre-sort for bulk submission was that some parameters were only selectively set and instead of using preset defaults, it would default to the value from a previous job within the same bulk submit. So bulk submits with some jobs setting the parameters and some jobs without setting the parameters lead to weird behavior.\r\n\r\nThis patch fixed that for the highIOjob flag (always set for either 0 or 1) and numberOfCores (always set, even for singlecore jobs which now specifically request 1 cpu). That in turn allows us to get rid of the pre-sorting by numberOfCores and highIOjob.\r\n\r\nI tested this change for numberOfCores with a bulk submit of 26 jobs all requesting between 4 and 8 cpus and it worked fine, the jobs running in Condor showed up the correct number of cpus it requested.\r\n\r\nThere are still some parameters that are selectively set, but since we don't pre-sort by them, nothing is changed to how things work now. There is a chance we might even be able to get rid sorting by sandbox and just bulk submit everything without caring about sandbox, but that needs more work and also will need us to address these last outstanding parameters that are selectively set.\r\n\r\nExample for how to deal with such a parameter would be what I did for MaxWallTime. Old code set it selectively, new code defaults to 12h if it is not set. Of course, if we say that the expected wall time should always be set for a job, we can also simplify this.\r\n\r\nI'll follow up in HN with the list of parameters. We need to make sure that they are always set, either by always having the information available to set it or by providing some sensible default.\r\n\r\n@amaltaro @ticoann , would appreciate feedback and comments on this patch (as it exists now).\r\n","created_at":"2016-06-10T19:34:09Z","updated_at":"2016-07-18T18:59:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"de8846ae8b829111864cea544edf02ccb78ef735","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/dmwm/WMCore/pulls/6925/commits","review_comments_url":"https://api.github.com/repos/dmwm/WMCore/pulls/6925/comments","review_comment_url":"https://api.github.com/repos/dmwm/WMCore/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dmwm/WMCore/issues/6925/comments","statuses_url":"https://api.github.com/repos/dmwm/WMCore/statuses/5a142510117d43e7f66fd723d142bf82b8c507c0","head":{"label":"hufnagel:submit-bulk-efficiency","ref":"submit-bulk-efficiency","sha":"5a142510117d43e7f66fd723d142bf82b8c507c0","user":{"login":"hufnagel","id":1798355,"avatar_url":"https://avatars.githubusercontent.com/u/1798355?v=3","gravatar_id":"","url":"https://api.github.com/users/hufnagel","html_url":"https://github.com/hufnagel","followers_url":"https://api.github.com/users/hufnagel/followers","following_url":"https://api.github.com/users/hufnagel/following{/other_user}","gists_url":"https://api.github.com/users/hufnagel/gists{/gist_id}","starred_url":"https://api.github.com/users/hufnagel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hufnagel/subscriptions","organizations_url":"https://api.github.com/users/hufnagel/orgs","repos_url":"https://api.github.com/users/hufnagel/repos","events_url":"https://api.github.com/users/hufnagel/events{/privacy}","received_events_url":"https://api.github.com/users/hufnagel/received_events","type":"User","site_admin":false},"repo":{"id":5087540,"name":"WMCore","full_name":"hufnagel/WMCore","owner":{"login":"hufnagel","id":1798355,"avatar_url":"https://avatars.githubusercontent.com/u/1798355?v=3","gravatar_id":"","url":"https://api.github.com/users/hufnagel","html_url":"https://github.com/hufnagel","followers_url":"https://api.github.com/users/hufnagel/followers","following_url":"https://api.github.com/users/hufnagel/following{/other_user}","gists_url":"https://api.github.com/users/hufnagel/gists{/gist_id}","starred_url":"https://api.github.com/users/hufnagel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hufnagel/subscriptions","organizations_url":"https://api.github.com/users/hufnagel/orgs","repos_url":"https://api.github.com/users/hufnagel/repos","events_url":"https://api.github.com/users/hufnagel/events{/privacy}","received_events_url":"https://api.github.com/users/hufnagel/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/hufnagel/WMCore","description":"WMCore","fork":true,"url":"https://api.github.com/repos/hufnagel/WMCore","forks_url":"https://api.github.com/repos/hufnagel/WMCore/forks","keys_url":"https://api.github.com/repos/hufnagel/WMCore/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hufnagel/WMCore/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hufnagel/WMCore/teams","hooks_url":"https://api.github.com/repos/hufnagel/WMCore/hooks","issue_events_url":"https://api.github.com/repos/hufnagel/WMCore/issues/events{/number}","events_url":"https://api.github.com/repos/hufnagel/WMCore/events","assignees_url":"https://api.github.com/repos/hufnagel/WMCore/assignees{/user}","branches_url":"https://api.github.com/repos/hufnagel/WMCore/branches{/branch}","tags_url":"https://api.github.com/repos/hufnagel/WMCore/tags","blobs_url":"https://api.github.com/repos/hufnagel/WMCore/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hufnagel/WMCore/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hufnagel/WMCore/git/refs{/sha}","trees_url":"https://api.github.com/repos/hufnagel/WMCore/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hufnagel/WMCore/statuses/{sha}","languages_url":"https://api.github.com/repos/hufnagel/WMCore/languages","stargazers_url":"https://api.github.com/repos/hufnagel/WMCore/stargazers","contributors_url":"https://api.github.com/repos/hufnagel/WMCore/contributors","subscribers_url":"https://api.github.com/repos/hufnagel/WMCore/subscribers","subscription_url":"https://api.github.com/repos/hufnagel/WMCore/subscription","commits_url":"https://api.github.com/repos/hufnagel/WMCore/commits{/sha}","git_commits_url":"https://api.github.com/repos/hufnagel/WMCore/git/commits{/sha}","comments_url":"https://api.github.com/repos/hufnagel/WMCore/comments{/number}","issue_comment_url":"https://api.github.com/repos/hufnagel/WMCore/issues/comments{/number}","contents_url":"https://api.github.com/repos/hufnagel/WMCore/contents/{+path}","compare_url":"https://api.github.com/repos/hufnagel/WMCore/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hufnagel/WMCore/merges","archive_url":"https://api.github.com/repos/hufnagel/WMCore/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hufnagel/WMCore/downloads","issues_url":"https://api.github.com/repos/hufnagel/WMCore/issues{/number}","pulls_url":"https://api.github.com/repos/hufnagel/WMCore/pulls{/number}","milestones_url":"https://api.github.com/repos/hufnagel/WMCore/milestones{/number}","notifications_url":"https://api.github.com/repos/hufnagel/WMCore/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hufnagel/WMCore/labels{/name}","releases_url":"https://api.github.com/repos/hufnagel/WMCore/releases{/id}","deployments_url":"https://api.github.com/repos/hufnagel/WMCore/deployments","created_at":"2012-07-17T20:23:34Z","updated_at":"2015-05-07T20:25:46Z","pushed_at":"2016-07-17T00:53:39Z","git_url":"git://github.com/hufnagel/WMCore.git","ssh_url":"git@github.com:hufnagel/WMCore.git","clone_url":"https://github.com/hufnagel/WMCore.git","svn_url":"https://github.com/hufnagel/WMCore","homepage":"https://svnweb.cern.ch/trac/CMSDMWM/","size":26719,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dmwm:master","ref":"master","sha":"8376255c8e223707e57d5c6d22eff35266fff653","user":{"login":"dmwm","id":988897,"avatar_url":"https://avatars.githubusercontent.com/u/988897?v=3","gravatar_id":"","url":"https://api.github.com/users/dmwm","html_url":"https://github.com/dmwm","followers_url":"https://api.github.com/users/dmwm/followers","following_url":"https://api.github.com/users/dmwm/following{/other_user}","gists_url":"https://api.github.com/users/dmwm/gists{/gist_id}","starred_url":"https://api.github.com/users/dmwm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dmwm/subscriptions","organizations_url":"https://api.github.com/users/dmwm/orgs","repos_url":"https://api.github.com/users/dmwm/repos","events_url":"https://api.github.com/users/dmwm/events{/privacy}","received_events_url":"https://api.github.com/users/dmwm/received_events","type":"Organization","site_admin":false},"repo":{"id":2370458,"name":"WMCore","full_name":"dmwm/WMCore","owner":{"login":"dmwm","id":988897,"avatar_url":"https://avatars.githubusercontent.com/u/988897?v=3","gravatar_id":"","url":"https://api.github.com/users/dmwm","html_url":"https://github.com/dmwm","followers_url":"https://api.github.com/users/dmwm/followers","following_url":"https://api.github.com/users/dmwm/following{/other_user}","gists_url":"https://api.github.com/users/dmwm/gists{/gist_id}","starred_url":"https://api.github.com/users/dmwm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dmwm/subscriptions","organizations_url":"https://api.github.com/users/dmwm/orgs","repos_url":"https://api.github.com/users/dmwm/repos","events_url":"https://api.github.com/users/dmwm/events{/privacy}","received_events_url":"https://api.github.com/users/dmwm/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dmwm/WMCore","description":"WMCore","fork":false,"url":"https://api.github.com/repos/dmwm/WMCore","forks_url":"https://api.github.com/repos/dmwm/WMCore/forks","keys_url":"https://api.github.com/repos/dmwm/WMCore/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dmwm/WMCore/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dmwm/WMCore/teams","hooks_url":"https://api.github.com/repos/dmwm/WMCore/hooks","issue_events_url":"https://api.github.com/repos/dmwm/WMCore/issues/events{/number}","events_url":"https://api.github.com/repos/dmwm/WMCore/events","assignees_url":"https://api.github.com/repos/dmwm/WMCore/assignees{/user}","branches_url":"https://api.github.com/repos/dmwm/WMCore/branches{/branch}","tags_url":"https://api.github.com/repos/dmwm/WMCore/tags","blobs_url":"https://api.github.com/repos/dmwm/WMCore/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dmwm/WMCore/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dmwm/WMCore/git/refs{/sha}","trees_url":"https://api.github.com/repos/dmwm/WMCore/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dmwm/WMCore/statuses/{sha}","languages_url":"https://api.github.com/repos/dmwm/WMCore/languages","stargazers_url":"https://api.github.com/repos/dmwm/WMCore/stargazers","contributors_url":"https://api.github.com/repos/dmwm/WMCore/contributors","subscribers_url":"https://api.github.com/repos/dmwm/WMCore/subscribers","subscription_url":"https://api.github.com/repos/dmwm/WMCore/subscription","commits_url":"https://api.github.com/repos/dmwm/WMCore/commits{/sha}","git_commits_url":"https://api.github.com/repos/dmwm/WMCore/git/commits{/sha}","comments_url":"https://api.github.com/repos/dmwm/WMCore/comments{/number}","issue_comment_url":"https://api.github.com/repos/dmwm/WMCore/issues/comments{/number}","contents_url":"https://api.github.com/repos/dmwm/WMCore/contents/{+path}","compare_url":"https://api.github.com/repos/dmwm/WMCore/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dmwm/WMCore/merges","archive_url":"https://api.github.com/repos/dmwm/WMCore/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dmwm/WMCore/downloads","issues_url":"https://api.github.com/repos/dmwm/WMCore/issues{/number}","pulls_url":"https://api.github.com/repos/dmwm/WMCore/pulls{/number}","milestones_url":"https://api.github.com/repos/dmwm/WMCore/milestones{/number}","notifications_url":"https://api.github.com/repos/dmwm/WMCore/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dmwm/WMCore/labels{/name}","releases_url":"https://api.github.com/repos/dmwm/WMCore/releases{/id}","deployments_url":"https://api.github.com/repos/dmwm/WMCore/deployments","created_at":"2011-09-12T10:37:21Z","updated_at":"2016-04-07T15:35:34Z","pushed_at":"2016-07-18T16:16:26Z","git_url":"git://github.com/dmwm/WMCore.git","ssh_url":"git@github.com:dmwm/WMCore.git","clone_url":"https://github.com/dmwm/WMCore.git","svn_url":"https://github.com/dmwm/WMCore","homepage":"https://svnweb.cern.ch/trac/CMSDMWM/","size":33794,"stargazers_count":24,"watchers_count":24,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":56,"mirror_url":null,"open_issues_count":403,"forks":56,"open_issues":403,"watchers":24,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/6925"},"html":{"href":"https://github.com/dmwm/WMCore/pull/6925"},"issue":{"href":"https://api.github.com/repos/dmwm/WMCore/issues/6925"},"comments":{"href":"https://api.github.com/repos/dmwm/WMCore/issues/6925/comments"},"review_comments":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/6925/comments"},"review_comment":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dmwm/WMCore/pulls/6925/commits"},"statuses":{"href":"https://api.github.com/repos/dmwm/WMCore/statuses/5a142510117d43e7f66fd723d142bf82b8c507c0"}}}}
{"actor":{"display_login":"amaltaro"}}