{"action":"created","comment":{"url":"https://api.github.com/repos/uzh/gc3pie/pulls/comments/77807699","id":77807699,"diff_hunk":"@@ -0,0 +1,211 @@\n+#! /usr/bin/env python\n+#\n+#   gtree.py -- Front-end script for running an R script\n+#   over a defined Data set on a given number of nodes .\n+#\n+#   Copyright (C) 2014, 2015 GC3, University of Zurich\n+#\n+#   This program is free software: you can redistribute it and/or\n+#   modify\n+#   it under the terms of the GNU General Public License as published by\n+#   the Free Software Foundation, either version 3 of the License, or\n+#   (at your option) any later version.\n+#\n+#   This program is distributed in the hope that it will be useful,\n+#   but WITHOUT ANY WARRANTY; without even the implied warranty of\n+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+#   GNU General Public License for more details.\n+#\n+#   You should have received a copy of the GNU General Public License\n+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.\n+#\n+\"\"\"\n+\n+It uses the generic `gc3libs.cmdline.SessionBasedScript` framework.\n+\n+See the output of ``gtree.py --help`` for program usage\n+instructions.\n+\n+Input parameters consists of:\n+\n+...\n+\n+Options:\n+\"\"\"\n+\n+__version__ = 'development version (SVN $Revision$)'\n+# summary of user-visible changes\n+__changelog__ = \"\"\"\n+  2016-05-11:\n+  * Initial version\n+\"\"\"\n+__author__ = 'Adrian Etter <adrian.etter@uzh.ch>'\n+__docformat__ = 'reStructuredText'\n+\n+\n+# run script, but allow GC3Pie persistence module to access classes defined here;\n+# for details, see: https://github.com/uzh/gc3pie/issues/95\n+if __name__ == \"__main__\":\n+    import gtree\n+    gtree.GtreeScript().run()\n+\n+from argparse import ArgumentError\n+import os\n+from os.path import basename, exists, join, realpath\n+import sys\n+import time\n+import tempfile\n+\n+import shutil\n+# import csv\n+\n+from pkg_resources import Requirement, resource_filename\n+\n+import gc3libs\n+import gc3libs.exceptions\n+from gc3libs import Application, Run, Task\n+from gc3libs.cmdline import SessionBasedScript, executable_file\n+import gc3libs.utils\n+from gc3libs.quantity import Memory, kB, MB, MiB, GB, Duration, hours, minutes, seconds\n+from gc3libs.workflow import RetryableTask\n+\n+DEFAULT_CORES = 4\n+DEFAULT_MEMORY = Memory(7,GB)\n+DEFAULT_WALLTIME = Duration(200,hours)\n+\n+## custom application class\n+class GtreeApplication(Application):\n+    \"\"\"\n+    Custom class to wrap the execution of a R script\n+    A convenient wrapper script is provided to facilitate how the command file\n+    is executed on the remote end.\n+    \"\"\"\n+    application_name = 'gtree'\n+\n+    def __init__(self, input_function, input_dataSet, **extra_args):\n+\n+        self.output_dir = extra_args['output_dir']\n+        self.input_function = input_function\n+        #\n+        inputs = dict()\n+        # outputs = dict()\n+        #\n+        # gtree_wrapper_sh = resource_filename(Requirement.parse(\"gc3pie\"),\n+        #                                   \"gc3libs/etc/gtree_wrapper.sh\")\n+        gtree_wrapper_sh = resource_filename(Requirement.parse(\"gc3pie\"),\n+                                              \"gc3libs/etc/gtree_wrapper.sh\")\n+        #\n+        inputs[gtree_wrapper_sh] = os.path.basename(gtree_wrapper_sh)\n+        # inputs['foo'] = 'foo'\n+        #\n+        inputs[input_function] = \"%s\" % os.path.basename(input_function)\n+        inputs[input_dataSet] = \"%s\" % os.path.basename(input_dataSet)\n+\n+        #\n+        arguments = \"./%s %s %s\" % (inputs[gtree_wrapper_sh], inputs[input_function], inputs[input_dataSet])\n+        # print 'hello'\n+        Application.__init__(\n+            self,\n+            arguments = arguments,\n+            inputs = inputs,\n+            outputs = [\"./results\"],\n+            stdout = 'gtree.log',\n+            join=True,\n+            executables = \"./%s\" % os.path.basename(input_function),\n+            **extra_args)\n+\n+    def terminated(self):\n+        \"\"\"\n+        Move results into original results folder\n+        \"\"\"\n+        gc3libs.log.info(\"Application terminated with exit code %s\" % self.execution.exitcode)\n+        for result in os.listdir(os.path.join(self.output_dir,\"results/\")):\n+            shutil.move(os.path.join(self.output_dir,\"results/\",result),\n+                        os.path.join(self.input_folder,\"results\",os.path.basename(result)))\n+        # Cleanup\n+        os.removedirs(os.path.join(self.output_dir,\"results/\"))\n+\n+class GtreeScript(SessionBasedScript):\n+    \"\"\"\n+    The script takes as input a rscript, the number of parallel jobs to run and the","path":"gc3apps/ivs.uzh/gtree.py","position":130,"original_position":130,"commit_id":"2e71ddf7c93725d5279cbbac3a8f4d61253eb94a","original_commit_id":"2e71ddf7c93725d5279cbbac3a8f4d61253eb94a","user":{"login":"riccardomurri","id":447228,"avatar_url":"https://avatars.githubusercontent.com/u/447228?v=3","gravatar_id":"","url":"https://api.github.com/users/riccardomurri","html_url":"https://github.com/riccardomurri","followers_url":"https://api.github.com/users/riccardomurri/followers","following_url":"https://api.github.com/users/riccardomurri/following{/other_user}","gists_url":"https://api.github.com/users/riccardomurri/gists{/gist_id}","starred_url":"https://api.github.com/users/riccardomurri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/riccardomurri/subscriptions","organizations_url":"https://api.github.com/users/riccardomurri/orgs","repos_url":"https://api.github.com/users/riccardomurri/repos","events_url":"https://api.github.com/users/riccardomurri/events{/privacy}","received_events_url":"https://api.github.com/users/riccardomurri/received_events","type":"User","site_admin":false},"body":"See comment re: help text in the GcsvBulkScript class.","created_at":"2016-09-07T11:58:32Z","updated_at":"2016-09-07T11:58:32Z","html_url":"https://github.com/uzh/gc3pie/pull/567#discussion_r77807699","pull_request_url":"https://api.github.com/repos/uzh/gc3pie/pulls/567","_links":{"self":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/comments/77807699"},"html":{"href":"https://github.com/uzh/gc3pie/pull/567#discussion_r77807699"},"pull_request":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/567"}}},"pull_request":{"url":"https://api.github.com/repos/uzh/gc3pie/pulls/567","id":81789653,"html_url":"https://github.com/uzh/gc3pie/pull/567","diff_url":"https://github.com/uzh/gc3pie/pull/567.diff","patch_url":"https://github.com/uzh/gc3pie/pull/567.patch","issue_url":"https://api.github.com/repos/uzh/gc3pie/issues/567","number":567,"state":"open","locked":false,"title":"Feature/gcsv bulk","user":{"login":"eydrian","id":535525,"avatar_url":"https://avatars.githubusercontent.com/u/535525?v=3","gravatar_id":"","url":"https://api.github.com/users/eydrian","html_url":"https://github.com/eydrian","followers_url":"https://api.github.com/users/eydrian/followers","following_url":"https://api.github.com/users/eydrian/following{/other_user}","gists_url":"https://api.github.com/users/eydrian/gists{/gist_id}","starred_url":"https://api.github.com/users/eydrian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/eydrian/subscriptions","organizations_url":"https://api.github.com/users/eydrian/orgs","repos_url":"https://api.github.com/users/eydrian/repos","events_url":"https://api.github.com/users/eydrian/events{/privacy}","received_events_url":"https://api.github.com/users/eydrian/received_events","type":"User","site_admin":false},"body":"add two new driver scripts:\r\n - `gcsvBulk` covers the issue 2380. It allows to submit a task, where a csv file can be split in multiple parts and each part will be processed on a distinct node by a specified matlab function.\r\n\r\n```\r\ngcsvBulk.py matlabScript.m  params.csv 4\r\n```\r\n\r\n- `gtree` covers the issue 1703. A given R script can be run on the specified number of nodes with no additional parameters given\r\n\r\n```\r\ngtree.py -N 100 dataset\r\n```","created_at":"2016-08-18T10:08:33Z","updated_at":"2016-09-07T11:58:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b1f63fb5705787db53849bf9b88f5ba1c3c0f1b4","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/uzh/gc3pie/pulls/567/commits","review_comments_url":"https://api.github.com/repos/uzh/gc3pie/pulls/567/comments","review_comment_url":"https://api.github.com/repos/uzh/gc3pie/pulls/comments{/number}","comments_url":"https://api.github.com/repos/uzh/gc3pie/issues/567/comments","statuses_url":"https://api.github.com/repos/uzh/gc3pie/statuses/2e71ddf7c93725d5279cbbac3a8f4d61253eb94a","head":{"label":"eydrian:feature/gcsvBulk","ref":"feature/gcsvBulk","sha":"2e71ddf7c93725d5279cbbac3a8f4d61253eb94a","user":{"login":"eydrian","id":535525,"avatar_url":"https://avatars.githubusercontent.com/u/535525?v=3","gravatar_id":"","url":"https://api.github.com/users/eydrian","html_url":"https://github.com/eydrian","followers_url":"https://api.github.com/users/eydrian/followers","following_url":"https://api.github.com/users/eydrian/following{/other_user}","gists_url":"https://api.github.com/users/eydrian/gists{/gist_id}","starred_url":"https://api.github.com/users/eydrian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/eydrian/subscriptions","organizations_url":"https://api.github.com/users/eydrian/orgs","repos_url":"https://api.github.com/users/eydrian/repos","events_url":"https://api.github.com/users/eydrian/events{/privacy}","received_events_url":"https://api.github.com/users/eydrian/received_events","type":"User","site_admin":false},"repo":{"id":59119030,"name":"gc3pie","full_name":"eydrian/gc3pie","owner":{"login":"eydrian","id":535525,"avatar_url":"https://avatars.githubusercontent.com/u/535525?v=3","gravatar_id":"","url":"https://api.github.com/users/eydrian","html_url":"https://github.com/eydrian","followers_url":"https://api.github.com/users/eydrian/followers","following_url":"https://api.github.com/users/eydrian/following{/other_user}","gists_url":"https://api.github.com/users/eydrian/gists{/gist_id}","starred_url":"https://api.github.com/users/eydrian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/eydrian/subscriptions","organizations_url":"https://api.github.com/users/eydrian/orgs","repos_url":"https://api.github.com/users/eydrian/repos","events_url":"https://api.github.com/users/eydrian/events{/privacy}","received_events_url":"https://api.github.com/users/eydrian/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/eydrian/gc3pie","description":"Python libraries and tools for running applications on diverse Grids and clusters","fork":true,"url":"https://api.github.com/repos/eydrian/gc3pie","forks_url":"https://api.github.com/repos/eydrian/gc3pie/forks","keys_url":"https://api.github.com/repos/eydrian/gc3pie/keys{/key_id}","collaborators_url":"https://api.github.com/repos/eydrian/gc3pie/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/eydrian/gc3pie/teams","hooks_url":"https://api.github.com/repos/eydrian/gc3pie/hooks","issue_events_url":"https://api.github.com/repos/eydrian/gc3pie/issues/events{/number}","events_url":"https://api.github.com/repos/eydrian/gc3pie/events","assignees_url":"https://api.github.com/repos/eydrian/gc3pie/assignees{/user}","branches_url":"https://api.github.com/repos/eydrian/gc3pie/branches{/branch}","tags_url":"https://api.github.com/repos/eydrian/gc3pie/tags","blobs_url":"https://api.github.com/repos/eydrian/gc3pie/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/eydrian/gc3pie/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/eydrian/gc3pie/git/refs{/sha}","trees_url":"https://api.github.com/repos/eydrian/gc3pie/git/trees{/sha}","statuses_url":"https://api.github.com/repos/eydrian/gc3pie/statuses/{sha}","languages_url":"https://api.github.com/repos/eydrian/gc3pie/languages","stargazers_url":"https://api.github.com/repos/eydrian/gc3pie/stargazers","contributors_url":"https://api.github.com/repos/eydrian/gc3pie/contributors","subscribers_url":"https://api.github.com/repos/eydrian/gc3pie/subscribers","subscription_url":"https://api.github.com/repos/eydrian/gc3pie/subscription","commits_url":"https://api.github.com/repos/eydrian/gc3pie/commits{/sha}","git_commits_url":"https://api.github.com/repos/eydrian/gc3pie/git/commits{/sha}","comments_url":"https://api.github.com/repos/eydrian/gc3pie/comments{/number}","issue_comment_url":"https://api.github.com/repos/eydrian/gc3pie/issues/comments{/number}","contents_url":"https://api.github.com/repos/eydrian/gc3pie/contents/{+path}","compare_url":"https://api.github.com/repos/eydrian/gc3pie/compare/{base}...{head}","merges_url":"https://api.github.com/repos/eydrian/gc3pie/merges","archive_url":"https://api.github.com/repos/eydrian/gc3pie/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/eydrian/gc3pie/downloads","issues_url":"https://api.github.com/repos/eydrian/gc3pie/issues{/number}","pulls_url":"https://api.github.com/repos/eydrian/gc3pie/pulls{/number}","milestones_url":"https://api.github.com/repos/eydrian/gc3pie/milestones{/number}","notifications_url":"https://api.github.com/repos/eydrian/gc3pie/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/eydrian/gc3pie/labels{/name}","releases_url":"https://api.github.com/repos/eydrian/gc3pie/releases{/id}","deployments_url":"https://api.github.com/repos/eydrian/gc3pie/deployments","created_at":"2016-05-18T13:40:16Z","updated_at":"2016-05-18T13:40:21Z","pushed_at":"2016-08-25T14:13:43Z","git_url":"git://github.com/eydrian/gc3pie.git","ssh_url":"git@github.com:eydrian/gc3pie.git","clone_url":"https://github.com/eydrian/gc3pie.git","svn_url":"https://github.com/eydrian/gc3pie","homepage":"http://gc3pie.readthedocs.org/","size":154213,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"uzh:master","ref":"master","sha":"b6125eb9dc7a605b918993a3a798bd71d543ab84","user":{"login":"uzh","id":902825,"avatar_url":"https://avatars.githubusercontent.com/u/902825?v=3","gravatar_id":"","url":"https://api.github.com/users/uzh","html_url":"https://github.com/uzh","followers_url":"https://api.github.com/users/uzh/followers","following_url":"https://api.github.com/users/uzh/following{/other_user}","gists_url":"https://api.github.com/users/uzh/gists{/gist_id}","starred_url":"https://api.github.com/users/uzh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uzh/subscriptions","organizations_url":"https://api.github.com/users/uzh/orgs","repos_url":"https://api.github.com/users/uzh/repos","events_url":"https://api.github.com/users/uzh/events{/privacy}","received_events_url":"https://api.github.com/users/uzh/received_events","type":"Organization","site_admin":false},"repo":{"id":40307352,"name":"gc3pie","full_name":"uzh/gc3pie","owner":{"login":"uzh","id":902825,"avatar_url":"https://avatars.githubusercontent.com/u/902825?v=3","gravatar_id":"","url":"https://api.github.com/users/uzh","html_url":"https://github.com/uzh","followers_url":"https://api.github.com/users/uzh/followers","following_url":"https://api.github.com/users/uzh/following{/other_user}","gists_url":"https://api.github.com/users/uzh/gists{/gist_id}","starred_url":"https://api.github.com/users/uzh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uzh/subscriptions","organizations_url":"https://api.github.com/users/uzh/orgs","repos_url":"https://api.github.com/users/uzh/repos","events_url":"https://api.github.com/users/uzh/events{/privacy}","received_events_url":"https://api.github.com/users/uzh/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uzh/gc3pie","description":"Python libraries and tools for running applications on diverse Grids and clusters","fork":false,"url":"https://api.github.com/repos/uzh/gc3pie","forks_url":"https://api.github.com/repos/uzh/gc3pie/forks","keys_url":"https://api.github.com/repos/uzh/gc3pie/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uzh/gc3pie/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uzh/gc3pie/teams","hooks_url":"https://api.github.com/repos/uzh/gc3pie/hooks","issue_events_url":"https://api.github.com/repos/uzh/gc3pie/issues/events{/number}","events_url":"https://api.github.com/repos/uzh/gc3pie/events","assignees_url":"https://api.github.com/repos/uzh/gc3pie/assignees{/user}","branches_url":"https://api.github.com/repos/uzh/gc3pie/branches{/branch}","tags_url":"https://api.github.com/repos/uzh/gc3pie/tags","blobs_url":"https://api.github.com/repos/uzh/gc3pie/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uzh/gc3pie/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uzh/gc3pie/git/refs{/sha}","trees_url":"https://api.github.com/repos/uzh/gc3pie/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uzh/gc3pie/statuses/{sha}","languages_url":"https://api.github.com/repos/uzh/gc3pie/languages","stargazers_url":"https://api.github.com/repos/uzh/gc3pie/stargazers","contributors_url":"https://api.github.com/repos/uzh/gc3pie/contributors","subscribers_url":"https://api.github.com/repos/uzh/gc3pie/subscribers","subscription_url":"https://api.github.com/repos/uzh/gc3pie/subscription","commits_url":"https://api.github.com/repos/uzh/gc3pie/commits{/sha}","git_commits_url":"https://api.github.com/repos/uzh/gc3pie/git/commits{/sha}","comments_url":"https://api.github.com/repos/uzh/gc3pie/comments{/number}","issue_comment_url":"https://api.github.com/repos/uzh/gc3pie/issues/comments{/number}","contents_url":"https://api.github.com/repos/uzh/gc3pie/contents/{+path}","compare_url":"https://api.github.com/repos/uzh/gc3pie/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uzh/gc3pie/merges","archive_url":"https://api.github.com/repos/uzh/gc3pie/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uzh/gc3pie/downloads","issues_url":"https://api.github.com/repos/uzh/gc3pie/issues{/number}","pulls_url":"https://api.github.com/repos/uzh/gc3pie/pulls{/number}","milestones_url":"https://api.github.com/repos/uzh/gc3pie/milestones{/number}","notifications_url":"https://api.github.com/repos/uzh/gc3pie/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uzh/gc3pie/labels{/name}","releases_url":"https://api.github.com/repos/uzh/gc3pie/releases{/id}","deployments_url":"https://api.github.com/repos/uzh/gc3pie/deployments","created_at":"2015-08-06T13:51:20Z","updated_at":"2016-07-29T19:55:53Z","pushed_at":"2016-09-06T12:39:08Z","git_url":"git://github.com/uzh/gc3pie.git","ssh_url":"git@github.com:uzh/gc3pie.git","clone_url":"https://github.com/uzh/gc3pie.git","svn_url":"https://github.com/uzh/gc3pie","homepage":"http://gc3pie.readthedocs.org/","size":167233,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":204,"forks":7,"open_issues":204,"watchers":10,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/567"},"html":{"href":"https://github.com/uzh/gc3pie/pull/567"},"issue":{"href":"https://api.github.com/repos/uzh/gc3pie/issues/567"},"comments":{"href":"https://api.github.com/repos/uzh/gc3pie/issues/567/comments"},"review_comments":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/567/comments"},"review_comment":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/uzh/gc3pie/pulls/567/commits"},"statuses":{"href":"https://api.github.com/repos/uzh/gc3pie/statuses/2e71ddf7c93725d5279cbbac3a8f4d61253eb94a"}}}}
{"actor":{"display_login":"riccardomurri"}}