{"action":"created","comment":{"url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/comments/61029576","id":61029576,"diff_hunk":"@@ -212,58 +161,199 @@ def upload_resources(src_dir, project=None, folder='/', ensure_upload=False):\n     if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0:\n         target_folder = applet_spec['folder'] if 'folder' in applet_spec else folder\n \n-        # Optimistically look for a resource bundle with the same\n-        # contents, and reuse it if possible. The resource bundle\n-        # carries a property 'resource_bundle_checksum' that indicates\n-        # the checksum; the way in which the checksum is computed is\n-        # given in the documentation of _directory_checksum.\n+        # Copy the code previously in _directory_checksum here and we will create\n+        # the tar file while performing the checksum.  If the checksum matches \n+        # (and ensure_upload is False), then we will use the existing file,\n+        # otherwise, we will compress and upload the tarball.\n+        \n+        with tempfile.NamedTemporaryFile(suffix=\".tar\") as tar_fh:\n+        \n+            output_sha1 = hashlib.sha1()\n+        \n+            for dirname, subdirs, files in os.walk(resources_dir):\n+                if not dirname.startswith(resources_dir):\n+                    raise AssertionError('Expected %r to start with root directory %r' % (dirname, resources_dir))\n+\n+                # Add an entry for the directory itself\n+                relative_dirname = dirname[len(resources_dir):]\n+                dir_stat = os.lstat(dirname)\n+                if not relative_dirname.startswith('/'):\n+                    relative_dirname = '/' + relative_dirname\n+\n+                fields = [relative_dirname, str(dir_stat.st_mode), str(int(dir_stat.st_mtime * 1000))]\n+                output_sha1.update(b''.join(s.encode('utf-8') + b'\\0' for s in fields))\n+                \n+                # add an entry in the tar file for the current directory, but\n+                # do not recurse!\n+                subprocess.check_call(['tar', '-C', resources_dir, '-rf', tar_fh.name, '--no-recursion', '.' + relative_dirname])\n+                \n+                # Canonicalize the order of subdirectories; this is the order in\n+                # which they will be visited by os.walk\n+                subdirs.sort()\n+                \n+                # check the subdirectories for symlinks.  We should throw an error\n+                # if there are any links that point outside of the directory (unless\n+                # --force-symlinks is given).  If a link is pointing internal to \n+                # the directory (or --force-symlinks is given), we should add it \n+                # as a file.\n+                for subdir_name in subdirs:\n+                    dir_path = os.path.join(dirname, subdir_name)\n+\n+                    # If we do have a symlink,                     \n+                    if os.path.islink(dir_path):\n+                        # Let's get the pointed-to path to ensure that it is\n+                        # still in the directory\n+                        link_target = os.readlink(dir_path)","path":"src/python/dxpy/app_builder.py","position":120,"original_position":120,"commit_id":"c03461752fb3c5c4fc62e432604e4750d205c4c1","original_commit_id":"c03461752fb3c5c4fc62e432604e4750d205c4c1","user":{"login":"psung","id":146557,"avatar_url":"https://avatars.githubusercontent.com/u/146557?v=3","gravatar_id":"","url":"https://api.github.com/users/psung","html_url":"https://github.com/psung","followers_url":"https://api.github.com/users/psung/followers","following_url":"https://api.github.com/users/psung/following{/other_user}","gists_url":"https://api.github.com/users/psung/gists{/gist_id}","starred_url":"https://api.github.com/users/psung/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/psung/subscriptions","organizations_url":"https://api.github.com/users/psung/orgs","repos_url":"https://api.github.com/users/psung/repos","events_url":"https://api.github.com/users/psung/events{/privacy}","received_events_url":"https://api.github.com/users/psung/received_events","type":"User","site_admin":false},"body":"Can this part be factored out into a method so as to unify it with the similar-looking code below?","created_at":"2016-04-26T05:10:30Z","updated_at":"2016-04-26T05:10:30Z","html_url":"https://github.com/dnanexus/dx-toolkit/pull/142#discussion_r61029576","pull_request_url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142","_links":{"self":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/comments/61029576"},"html":{"href":"https://github.com/dnanexus/dx-toolkit/pull/142#discussion_r61029576"},"pull_request":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142"}}},"pull_request":{"url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142","id":67386760,"html_url":"https://github.com/dnanexus/dx-toolkit/pull/142","diff_url":"https://github.com/dnanexus/dx-toolkit/pull/142.diff","patch_url":"https://github.com/dnanexus/dx-toolkit/pull/142.patch","issue_url":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/142","number":142,"state":"open","locked":false,"title":"Added ability to dereference symlinks","user":{"login":"johnwallace123","id":1088080,"avatar_url":"https://avatars.githubusercontent.com/u/1088080?v=3","gravatar_id":"","url":"https://api.github.com/users/johnwallace123","html_url":"https://github.com/johnwallace123","followers_url":"https://api.github.com/users/johnwallace123/followers","following_url":"https://api.github.com/users/johnwallace123/following{/other_user}","gists_url":"https://api.github.com/users/johnwallace123/gists{/gist_id}","starred_url":"https://api.github.com/users/johnwallace123/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnwallace123/subscriptions","organizations_url":"https://api.github.com/users/johnwallace123/orgs","repos_url":"https://api.github.com/users/johnwallace123/repos","events_url":"https://api.github.com/users/johnwallace123/events{/privacy}","received_events_url":"https://api.github.com/users/johnwallace123/received_events","type":"User","site_admin":false},"body":"Adding the ability to dereference symbolic links that are pointing to\r\nremote files (i.e. not in the resource directory) of an app.  This patch\r\nwill ensure the following cases:\r\n\r\n1. Links to local files and directories are kept\r\n2. Links to remote files are dereferenced\r\n3. Broken links cause an AppBuilderError\r\n4. Links to remote directories cause an AppBuilderError\r\n5. Links to local links (regardless of destination) are kept\r\n6. Links to remote links (links to files) are dereferenced\r\n7. Absolute links are treated as remote links (case 2,4)\r\n8. Links that travel outside of the resource directory (i.e. ../resources/foo)\r\nare treated as remote links (case 2,4).  This is pretty evil; please\r\ndon't be evil.\r\n\r\nIn any case, adding the new option \"--force-symlinks\" to the \"dx build\"\r\ncommand will restore the previous behavior and keep symlinks that may or\r\nmay not resolve.  The use case for this could be perhaps a link in\r\nresources that will point to a directory known to exist in a DNAnexus\r\ninstance (like /home/dnanexus for example).\r\n\r\nNote that in the upload_resources_advanced test relies on the \"old\"\r\nbehavior of symlinks, so I also added the \"--force-symlinks\" tag to\r\nthose unit tests.","created_at":"2016-04-21T16:48:05Z","updated_at":"2016-04-26T05:10:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bfdc95fa342aabac4ed9eb6c60aa628da094ebd1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142/commits","review_comments_url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142/comments","review_comment_url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/142/comments","statuses_url":"https://api.github.com/repos/dnanexus/dx-toolkit/statuses/c03461752fb3c5c4fc62e432604e4750d205c4c1","head":{"label":"johnwallace123:localSymlinkOnly","ref":"localSymlinkOnly","sha":"c03461752fb3c5c4fc62e432604e4750d205c4c1","user":{"login":"johnwallace123","id":1088080,"avatar_url":"https://avatars.githubusercontent.com/u/1088080?v=3","gravatar_id":"","url":"https://api.github.com/users/johnwallace123","html_url":"https://github.com/johnwallace123","followers_url":"https://api.github.com/users/johnwallace123/followers","following_url":"https://api.github.com/users/johnwallace123/following{/other_user}","gists_url":"https://api.github.com/users/johnwallace123/gists{/gist_id}","starred_url":"https://api.github.com/users/johnwallace123/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnwallace123/subscriptions","organizations_url":"https://api.github.com/users/johnwallace123/orgs","repos_url":"https://api.github.com/users/johnwallace123/repos","events_url":"https://api.github.com/users/johnwallace123/events{/privacy}","received_events_url":"https://api.github.com/users/johnwallace123/received_events","type":"User","site_admin":false},"repo":{"id":40357958,"name":"dx-toolkit","full_name":"johnwallace123/dx-toolkit","owner":{"login":"johnwallace123","id":1088080,"avatar_url":"https://avatars.githubusercontent.com/u/1088080?v=3","gravatar_id":"","url":"https://api.github.com/users/johnwallace123","html_url":"https://github.com/johnwallace123","followers_url":"https://api.github.com/users/johnwallace123/followers","following_url":"https://api.github.com/users/johnwallace123/following{/other_user}","gists_url":"https://api.github.com/users/johnwallace123/gists{/gist_id}","starred_url":"https://api.github.com/users/johnwallace123/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnwallace123/subscriptions","organizations_url":"https://api.github.com/users/johnwallace123/orgs","repos_url":"https://api.github.com/users/johnwallace123/repos","events_url":"https://api.github.com/users/johnwallace123/events{/privacy}","received_events_url":"https://api.github.com/users/johnwallace123/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/johnwallace123/dx-toolkit","description":"DNAnexus platform client libraries.","fork":true,"url":"https://api.github.com/repos/johnwallace123/dx-toolkit","forks_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/forks","keys_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/keys{/key_id}","collaborators_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/teams","hooks_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/hooks","issue_events_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/issues/events{/number}","events_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/events","assignees_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/assignees{/user}","branches_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/branches{/branch}","tags_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/tags","blobs_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/git/refs{/sha}","trees_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/git/trees{/sha}","statuses_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/statuses/{sha}","languages_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/languages","stargazers_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/stargazers","contributors_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/contributors","subscribers_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/subscribers","subscription_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/subscription","commits_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/commits{/sha}","git_commits_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/git/commits{/sha}","comments_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/comments{/number}","issue_comment_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/issues/comments{/number}","contents_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/contents/{+path}","compare_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/compare/{base}...{head}","merges_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/merges","archive_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/downloads","issues_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/issues{/number}","pulls_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/pulls{/number}","milestones_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/milestones{/number}","notifications_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/labels{/name}","releases_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/releases{/id}","deployments_url":"https://api.github.com/repos/johnwallace123/dx-toolkit/deployments","created_at":"2015-08-07T11:51:33Z","updated_at":"2016-04-12T16:24:59Z","pushed_at":"2016-04-21T16:45:38Z","git_url":"git://github.com/johnwallace123/dx-toolkit.git","ssh_url":"git@github.com:johnwallace123/dx-toolkit.git","clone_url":"https://github.com/johnwallace123/dx-toolkit.git","svn_url":"https://github.com/johnwallace123/dx-toolkit","homepage":"http://autodoc.dnanexus.com/","size":18651,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dnanexus:master","ref":"master","sha":"4c5f29f5b2557d16a7b109104c8afcbfeaacd7cb","user":{"login":"dnanexus","id":1161297,"avatar_url":"https://avatars.githubusercontent.com/u/1161297?v=3","gravatar_id":"","url":"https://api.github.com/users/dnanexus","html_url":"https://github.com/dnanexus","followers_url":"https://api.github.com/users/dnanexus/followers","following_url":"https://api.github.com/users/dnanexus/following{/other_user}","gists_url":"https://api.github.com/users/dnanexus/gists{/gist_id}","starred_url":"https://api.github.com/users/dnanexus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dnanexus/subscriptions","organizations_url":"https://api.github.com/users/dnanexus/orgs","repos_url":"https://api.github.com/users/dnanexus/repos","events_url":"https://api.github.com/users/dnanexus/events{/privacy}","received_events_url":"https://api.github.com/users/dnanexus/received_events","type":"Organization","site_admin":false},"repo":{"id":4802269,"name":"dx-toolkit","full_name":"dnanexus/dx-toolkit","owner":{"login":"dnanexus","id":1161297,"avatar_url":"https://avatars.githubusercontent.com/u/1161297?v=3","gravatar_id":"","url":"https://api.github.com/users/dnanexus","html_url":"https://github.com/dnanexus","followers_url":"https://api.github.com/users/dnanexus/followers","following_url":"https://api.github.com/users/dnanexus/following{/other_user}","gists_url":"https://api.github.com/users/dnanexus/gists{/gist_id}","starred_url":"https://api.github.com/users/dnanexus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dnanexus/subscriptions","organizations_url":"https://api.github.com/users/dnanexus/orgs","repos_url":"https://api.github.com/users/dnanexus/repos","events_url":"https://api.github.com/users/dnanexus/events{/privacy}","received_events_url":"https://api.github.com/users/dnanexus/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dnanexus/dx-toolkit","description":"DNAnexus platform client libraries","fork":false,"url":"https://api.github.com/repos/dnanexus/dx-toolkit","forks_url":"https://api.github.com/repos/dnanexus/dx-toolkit/forks","keys_url":"https://api.github.com/repos/dnanexus/dx-toolkit/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dnanexus/dx-toolkit/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dnanexus/dx-toolkit/teams","hooks_url":"https://api.github.com/repos/dnanexus/dx-toolkit/hooks","issue_events_url":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/events{/number}","events_url":"https://api.github.com/repos/dnanexus/dx-toolkit/events","assignees_url":"https://api.github.com/repos/dnanexus/dx-toolkit/assignees{/user}","branches_url":"https://api.github.com/repos/dnanexus/dx-toolkit/branches{/branch}","tags_url":"https://api.github.com/repos/dnanexus/dx-toolkit/tags","blobs_url":"https://api.github.com/repos/dnanexus/dx-toolkit/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dnanexus/dx-toolkit/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dnanexus/dx-toolkit/git/refs{/sha}","trees_url":"https://api.github.com/repos/dnanexus/dx-toolkit/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dnanexus/dx-toolkit/statuses/{sha}","languages_url":"https://api.github.com/repos/dnanexus/dx-toolkit/languages","stargazers_url":"https://api.github.com/repos/dnanexus/dx-toolkit/stargazers","contributors_url":"https://api.github.com/repos/dnanexus/dx-toolkit/contributors","subscribers_url":"https://api.github.com/repos/dnanexus/dx-toolkit/subscribers","subscription_url":"https://api.github.com/repos/dnanexus/dx-toolkit/subscription","commits_url":"https://api.github.com/repos/dnanexus/dx-toolkit/commits{/sha}","git_commits_url":"https://api.github.com/repos/dnanexus/dx-toolkit/git/commits{/sha}","comments_url":"https://api.github.com/repos/dnanexus/dx-toolkit/comments{/number}","issue_comment_url":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/comments{/number}","contents_url":"https://api.github.com/repos/dnanexus/dx-toolkit/contents/{+path}","compare_url":"https://api.github.com/repos/dnanexus/dx-toolkit/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dnanexus/dx-toolkit/merges","archive_url":"https://api.github.com/repos/dnanexus/dx-toolkit/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dnanexus/dx-toolkit/downloads","issues_url":"https://api.github.com/repos/dnanexus/dx-toolkit/issues{/number}","pulls_url":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls{/number}","milestones_url":"https://api.github.com/repos/dnanexus/dx-toolkit/milestones{/number}","notifications_url":"https://api.github.com/repos/dnanexus/dx-toolkit/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dnanexus/dx-toolkit/labels{/name}","releases_url":"https://api.github.com/repos/dnanexus/dx-toolkit/releases{/id}","deployments_url":"https://api.github.com/repos/dnanexus/dx-toolkit/deployments","created_at":"2012-06-27T00:23:47Z","updated_at":"2016-04-25T15:27:41Z","pushed_at":"2016-04-25T22:54:14Z","git_url":"git://github.com/dnanexus/dx-toolkit.git","ssh_url":"git@github.com:dnanexus/dx-toolkit.git","clone_url":"https://github.com/dnanexus/dx-toolkit.git","svn_url":"https://github.com/dnanexus/dx-toolkit","homepage":"http://autodoc.dnanexus.com/","size":18641,"stargazers_count":33,"watchers_count":33,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":29,"mirror_url":null,"open_issues_count":9,"forks":29,"open_issues":9,"watchers":33,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142"},"html":{"href":"https://github.com/dnanexus/dx-toolkit/pull/142"},"issue":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/142"},"comments":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/issues/142/comments"},"review_comments":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142/comments"},"review_comment":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/pulls/142/commits"},"statuses":{"href":"https://api.github.com/repos/dnanexus/dx-toolkit/statuses/c03461752fb3c5c4fc62e432604e4750d205c4c1"}}}}