{"action":"created","comment":{"url":"https://api.github.com/repos/pantsbuild/pants/pulls/comments/169487698","pull_request_review_id":98021094,"id":169487698,"diff_hunk":"@@ -0,0 +1,64 @@\n+# coding=utf-8\n+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).\n+# Licensed under the Apache License, Version 2.0 (see LICENSE).\n+\n+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,\n+                        unicode_literals, with_statement)\n+\n+import os\n+\n+from pex.interpreter import PythonInterpreter\n+\n+from pants.util.memo import memoized_method\n+\n+\n+class SandboxedInterpreter(PythonInterpreter):\n+\n+  class ToolchainLocationError(Exception):\n+    def __init__(self, dir_path):\n+      msg = \"path '{}' does not exist or is not a directory\".format(dir_path)\n+      super(ToolchainLocationError, self).__init__(msg)\n+\n+  class BaseInterpreterError(Exception): pass\n+\n+  # using another PythonInterpreter to populate the superclass constructor args\n+  def __init__(self, llvm_base_dir, base_interp):\n+\n+    if not os.path.isdir(llvm_base_dir):\n+      raise ToolchainLocationError(llvm_base_dir)\n+    if not isinstance(base_interp, PythonInterpreter):\n+      raise BaseInterpreterError(\n+        \"invalid PythonInterpreter: '{}'\".format(repr(base_interp)))\n+\n+    self._llvm_base_dir = llvm_base_dir\n+\n+    # this feels a little hacky -- what if pex's PythonInterpreter later needs\n+    # another constructor arg that's not just a property of the class?\n+    super(SandboxedInterpreter, self).__init__(\n+      base_interp.binary, base_interp.identity, extras=base_interp.extras)\n+\n+  # made into an instance method here (unlike PythonInterpreter superclass) to\n+  # use instance property self._llvm_base_dir\n+  @memoized_method\n+  def sanitized_environment(self):\n+    sanitized_env = super(SandboxedInterpreter, self).sanitized_environment()\n+\n+    # use our compiler at the front of the path\n+    # TODO: when we provide ld and stdlib headers, don't add the original path\n+    sanitized_env['PATH'] = ':'.join([\n+      os.path.join(self._llvm_base_dir, 'bin'),\n+      os.environ.get('PATH'),\n+    ])\n+\n+    # TODO: figure out whether we actually should be compiling fat binaries.\n+    # this line tells distutils to only compile for 64-bit archs -- if not, it\n+    # will attempt to build a fat binary for 32- and 64-bit archs, which makes\n+    # clang invoke \"lipo\", an osx command which does not appear to be open\n+    # source. see Lib/distutils/sysconfig.py and Lib/_osx_support.py in CPython.\n+    sanitized_env['ARCHFLAGS'] = '-arch x86_64'","path":"src/python/pants/backend/python/sandboxed_interpreter.py","position":58,"original_position":58,"commit_id":"41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d","original_commit_id":"41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d","user":{"login":"stuhood","id":46740,"avatar_url":"https://avatars1.githubusercontent.com/u/46740?v=4","gravatar_id":"","url":"https://api.github.com/users/stuhood","html_url":"https://github.com/stuhood","followers_url":"https://api.github.com/users/stuhood/followers","following_url":"https://api.github.com/users/stuhood/following{/other_user}","gists_url":"https://api.github.com/users/stuhood/gists{/gist_id}","starred_url":"https://api.github.com/users/stuhood/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stuhood/subscriptions","organizations_url":"https://api.github.com/users/stuhood/orgs","repos_url":"https://api.github.com/users/stuhood/repos","events_url":"https://api.github.com/users/stuhood/events{/privacy}","received_events_url":"https://api.github.com/users/stuhood/received_events","type":"User","site_admin":false},"body":"Maybe I'm not understanding which things will run in this sandbox, but this seems specific to a particular usage of this class (for invoking `setup.py`). Would it be better for `build_local_python_distributions.py` to be able to set additional env vars?","created_at":"2018-02-20T22:55:48Z","updated_at":"2018-02-20T22:56:52Z","html_url":"https://github.com/pantsbuild/pants/pull/5490#discussion_r169487698","pull_request_url":"https://api.github.com/repos/pantsbuild/pants/pulls/5490","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/comments/169487698"},"html":{"href":"https://github.com/pantsbuild/pants/pull/5490#discussion_r169487698"},"pull_request":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/5490"}}},"pull_request":{"url":"https://api.github.com/repos/pantsbuild/pants/pulls/5490","id":170329276,"html_url":"https://github.com/pantsbuild/pants/pull/5490","diff_url":"https://github.com/pantsbuild/pants/pull/5490.diff","patch_url":"https://github.com/pantsbuild/pants/pull/5490.patch","issue_url":"https://api.github.com/repos/pantsbuild/pants/issues/5490","number":5490,"state":"open","locked":false,"title":"use the LLVM subsystem to invoke a provided clang to compile native code in python_dist()","user":{"login":"cosmicexplorer","id":1305167,"avatar_url":"https://avatars1.githubusercontent.com/u/1305167?v=4","gravatar_id":"","url":"https://api.github.com/users/cosmicexplorer","html_url":"https://github.com/cosmicexplorer","followers_url":"https://api.github.com/users/cosmicexplorer/followers","following_url":"https://api.github.com/users/cosmicexplorer/following{/other_user}","gists_url":"https://api.github.com/users/cosmicexplorer/gists{/gist_id}","starred_url":"https://api.github.com/users/cosmicexplorer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cosmicexplorer/subscriptions","organizations_url":"https://api.github.com/users/cosmicexplorer/orgs","repos_url":"https://api.github.com/users/cosmicexplorer/repos","events_url":"https://api.github.com/users/cosmicexplorer/events{/privacy}","received_events_url":"https://api.github.com/users/cosmicexplorer/received_events","type":"User","site_admin":false},"body":"### Problem\r\n\r\n`python_dist()` allows the use of native extension modules, but it currently uses whatever compiler, linker, libraries, *etc* that distutils can find first on the host to compile the native code. Providing these disparate elements of the native code compilation toolchain in Pants means users don't have to install it themselves, and Pants can ensure that the provided compiler supports whatever features we want to provide to developers of native extension modules with `python_dist()`.\r\n\r\n### Solution\r\n\r\nThis PR uses the LLVM subsystem defined in #5471 to compile native code in a setup.py-based project declared through a `python_dist()` target. The environment of the setup.py invocation was controlled by subclassing `pex.interpreter.PythonInterpreter` and overriding the `sanitized_environment()` method. This couples the compilation environment we create to the execution of the setup.py for a `python_dist()` target in (I think) a very natural way.\r\n\r\n### Result\r\n\r\nWhen a `python_dist()` target is built through the `BuildLocalPythonDistributions` task, it pulls in a bootstrapped LLVM toolchain **(currently just the clang compiler)** and prepends it to the `PATH` so that our provided compiler is preferred (in addition to scrubbing `CC` and `CXX`). True sandboxed execution is preferable, but for now this at least guarantees that the provided compiler is used, which satisfies the goals stated at the top.\r\n\r\nThe LLVM archive that Pants will attempt to unpack will be produced by a cross-platform script. The current version of this script that I am using for development is located [here](https://github.com/cosmicexplorer/binaries/blob/dmcclanahan/add-llvm-package-script/build-llvm-5.0.1.sh) -- it's very simple and works, but I don't know if that should be made available in S3 before this PR gets merged.","created_at":"2018-02-20T22:40:07Z","updated_at":"2018-02-20T22:56:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d8ebcf54fd9fe12e7b7db747ae7b5020cb86b45c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/pantsbuild/pants/pulls/5490/commits","review_comments_url":"https://api.github.com/repos/pantsbuild/pants/pulls/5490/comments","review_comment_url":"https://api.github.com/repos/pantsbuild/pants/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pantsbuild/pants/issues/5490/comments","statuses_url":"https://api.github.com/repos/pantsbuild/pants/statuses/41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d","head":{"label":"cosmicexplorer:dmcclanahan/python-dist-c++-sources","ref":"dmcclanahan/python-dist-c++-sources","sha":"41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d","user":{"login":"cosmicexplorer","id":1305167,"avatar_url":"https://avatars1.githubusercontent.com/u/1305167?v=4","gravatar_id":"","url":"https://api.github.com/users/cosmicexplorer","html_url":"https://github.com/cosmicexplorer","followers_url":"https://api.github.com/users/cosmicexplorer/followers","following_url":"https://api.github.com/users/cosmicexplorer/following{/other_user}","gists_url":"https://api.github.com/users/cosmicexplorer/gists{/gist_id}","starred_url":"https://api.github.com/users/cosmicexplorer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cosmicexplorer/subscriptions","organizations_url":"https://api.github.com/users/cosmicexplorer/orgs","repos_url":"https://api.github.com/users/cosmicexplorer/repos","events_url":"https://api.github.com/users/cosmicexplorer/events{/privacy}","received_events_url":"https://api.github.com/users/cosmicexplorer/received_events","type":"User","site_admin":false},"repo":{"id":59165183,"name":"pants","full_name":"cosmicexplorer/pants","owner":{"login":"cosmicexplorer","id":1305167,"avatar_url":"https://avatars1.githubusercontent.com/u/1305167?v=4","gravatar_id":"","url":"https://api.github.com/users/cosmicexplorer","html_url":"https://github.com/cosmicexplorer","followers_url":"https://api.github.com/users/cosmicexplorer/followers","following_url":"https://api.github.com/users/cosmicexplorer/following{/other_user}","gists_url":"https://api.github.com/users/cosmicexplorer/gists{/gist_id}","starred_url":"https://api.github.com/users/cosmicexplorer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cosmicexplorer/subscriptions","organizations_url":"https://api.github.com/users/cosmicexplorer/orgs","repos_url":"https://api.github.com/users/cosmicexplorer/repos","events_url":"https://api.github.com/users/cosmicexplorer/events{/privacy}","received_events_url":"https://api.github.com/users/cosmicexplorer/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cosmicexplorer/pants","description":"Pants Build System","fork":true,"url":"https://api.github.com/repos/cosmicexplorer/pants","forks_url":"https://api.github.com/repos/cosmicexplorer/pants/forks","keys_url":"https://api.github.com/repos/cosmicexplorer/pants/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cosmicexplorer/pants/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cosmicexplorer/pants/teams","hooks_url":"https://api.github.com/repos/cosmicexplorer/pants/hooks","issue_events_url":"https://api.github.com/repos/cosmicexplorer/pants/issues/events{/number}","events_url":"https://api.github.com/repos/cosmicexplorer/pants/events","assignees_url":"https://api.github.com/repos/cosmicexplorer/pants/assignees{/user}","branches_url":"https://api.github.com/repos/cosmicexplorer/pants/branches{/branch}","tags_url":"https://api.github.com/repos/cosmicexplorer/pants/tags","blobs_url":"https://api.github.com/repos/cosmicexplorer/pants/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cosmicexplorer/pants/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cosmicexplorer/pants/git/refs{/sha}","trees_url":"https://api.github.com/repos/cosmicexplorer/pants/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cosmicexplorer/pants/statuses/{sha}","languages_url":"https://api.github.com/repos/cosmicexplorer/pants/languages","stargazers_url":"https://api.github.com/repos/cosmicexplorer/pants/stargazers","contributors_url":"https://api.github.com/repos/cosmicexplorer/pants/contributors","subscribers_url":"https://api.github.com/repos/cosmicexplorer/pants/subscribers","subscription_url":"https://api.github.com/repos/cosmicexplorer/pants/subscription","commits_url":"https://api.github.com/repos/cosmicexplorer/pants/commits{/sha}","git_commits_url":"https://api.github.com/repos/cosmicexplorer/pants/git/commits{/sha}","comments_url":"https://api.github.com/repos/cosmicexplorer/pants/comments{/number}","issue_comment_url":"https://api.github.com/repos/cosmicexplorer/pants/issues/comments{/number}","contents_url":"https://api.github.com/repos/cosmicexplorer/pants/contents/{+path}","compare_url":"https://api.github.com/repos/cosmicexplorer/pants/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cosmicexplorer/pants/merges","archive_url":"https://api.github.com/repos/cosmicexplorer/pants/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cosmicexplorer/pants/downloads","issues_url":"https://api.github.com/repos/cosmicexplorer/pants/issues{/number}","pulls_url":"https://api.github.com/repos/cosmicexplorer/pants/pulls{/number}","milestones_url":"https://api.github.com/repos/cosmicexplorer/pants/milestones{/number}","notifications_url":"https://api.github.com/repos/cosmicexplorer/pants/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cosmicexplorer/pants/labels{/name}","releases_url":"https://api.github.com/repos/cosmicexplorer/pants/releases{/id}","deployments_url":"https://api.github.com/repos/cosmicexplorer/pants/deployments","created_at":"2016-05-19T01:53:21Z","updated_at":"2016-05-19T01:53:26Z","pushed_at":"2018-02-20T22:18:05Z","git_url":"git://github.com/cosmicexplorer/pants.git","ssh_url":"git@github.com:cosmicexplorer/pants.git","clone_url":"https://github.com/cosmicexplorer/pants.git","svn_url":"https://github.com/cosmicexplorer/pants","homepage":"http://pantsbuild.github.io/","size":82714,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pantsbuild:master","ref":"master","sha":"71a33d65d11458ccceb67126c9dc418682e479a5","user":{"login":"pantsbuild","id":3065172,"avatar_url":"https://avatars0.githubusercontent.com/u/3065172?v=4","gravatar_id":"","url":"https://api.github.com/users/pantsbuild","html_url":"https://github.com/pantsbuild","followers_url":"https://api.github.com/users/pantsbuild/followers","following_url":"https://api.github.com/users/pantsbuild/following{/other_user}","gists_url":"https://api.github.com/users/pantsbuild/gists{/gist_id}","starred_url":"https://api.github.com/users/pantsbuild/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pantsbuild/subscriptions","organizations_url":"https://api.github.com/users/pantsbuild/orgs","repos_url":"https://api.github.com/users/pantsbuild/repos","events_url":"https://api.github.com/users/pantsbuild/events{/privacy}","received_events_url":"https://api.github.com/users/pantsbuild/received_events","type":"Organization","site_admin":false},"repo":{"id":7209075,"name":"pants","full_name":"pantsbuild/pants","owner":{"login":"pantsbuild","id":3065172,"avatar_url":"https://avatars0.githubusercontent.com/u/3065172?v=4","gravatar_id":"","url":"https://api.github.com/users/pantsbuild","html_url":"https://github.com/pantsbuild","followers_url":"https://api.github.com/users/pantsbuild/followers","following_url":"https://api.github.com/users/pantsbuild/following{/other_user}","gists_url":"https://api.github.com/users/pantsbuild/gists{/gist_id}","starred_url":"https://api.github.com/users/pantsbuild/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pantsbuild/subscriptions","organizations_url":"https://api.github.com/users/pantsbuild/orgs","repos_url":"https://api.github.com/users/pantsbuild/repos","events_url":"https://api.github.com/users/pantsbuild/events{/privacy}","received_events_url":"https://api.github.com/users/pantsbuild/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pantsbuild/pants","description":"The Pants Build System","fork":false,"url":"https://api.github.com/repos/pantsbuild/pants","forks_url":"https://api.github.com/repos/pantsbuild/pants/forks","keys_url":"https://api.github.com/repos/pantsbuild/pants/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pantsbuild/pants/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pantsbuild/pants/teams","hooks_url":"https://api.github.com/repos/pantsbuild/pants/hooks","issue_events_url":"https://api.github.com/repos/pantsbuild/pants/issues/events{/number}","events_url":"https://api.github.com/repos/pantsbuild/pants/events","assignees_url":"https://api.github.com/repos/pantsbuild/pants/assignees{/user}","branches_url":"https://api.github.com/repos/pantsbuild/pants/branches{/branch}","tags_url":"https://api.github.com/repos/pantsbuild/pants/tags","blobs_url":"https://api.github.com/repos/pantsbuild/pants/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pantsbuild/pants/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pantsbuild/pants/git/refs{/sha}","trees_url":"https://api.github.com/repos/pantsbuild/pants/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pantsbuild/pants/statuses/{sha}","languages_url":"https://api.github.com/repos/pantsbuild/pants/languages","stargazers_url":"https://api.github.com/repos/pantsbuild/pants/stargazers","contributors_url":"https://api.github.com/repos/pantsbuild/pants/contributors","subscribers_url":"https://api.github.com/repos/pantsbuild/pants/subscribers","subscription_url":"https://api.github.com/repos/pantsbuild/pants/subscription","commits_url":"https://api.github.com/repos/pantsbuild/pants/commits{/sha}","git_commits_url":"https://api.github.com/repos/pantsbuild/pants/git/commits{/sha}","comments_url":"https://api.github.com/repos/pantsbuild/pants/comments{/number}","issue_comment_url":"https://api.github.com/repos/pantsbuild/pants/issues/comments{/number}","contents_url":"https://api.github.com/repos/pantsbuild/pants/contents/{+path}","compare_url":"https://api.github.com/repos/pantsbuild/pants/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pantsbuild/pants/merges","archive_url":"https://api.github.com/repos/pantsbuild/pants/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pantsbuild/pants/downloads","issues_url":"https://api.github.com/repos/pantsbuild/pants/issues{/number}","pulls_url":"https://api.github.com/repos/pantsbuild/pants/pulls{/number}","milestones_url":"https://api.github.com/repos/pantsbuild/pants/milestones{/number}","notifications_url":"https://api.github.com/repos/pantsbuild/pants/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pantsbuild/pants/labels{/name}","releases_url":"https://api.github.com/repos/pantsbuild/pants/releases{/id}","deployments_url":"https://api.github.com/repos/pantsbuild/pants/deployments","created_at":"2012-12-17T17:39:04Z","updated_at":"2018-02-20T12:19:45Z","pushed_at":"2018-02-20T22:40:08Z","git_url":"git://github.com/pantsbuild/pants.git","ssh_url":"git@github.com:pantsbuild/pants.git","clone_url":"https://github.com/pantsbuild/pants.git","svn_url":"https://github.com/pantsbuild/pants","homepage":"http://pantsbuild.org/","size":82545,"stargazers_count":918,"watchers_count":918,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":258,"mirror_url":null,"archived":false,"open_issues_count":594,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":258,"open_issues":594,"watchers":918,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/5490"},"html":{"href":"https://github.com/pantsbuild/pants/pull/5490"},"issue":{"href":"https://api.github.com/repos/pantsbuild/pants/issues/5490"},"comments":{"href":"https://api.github.com/repos/pantsbuild/pants/issues/5490/comments"},"review_comments":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/5490/comments"},"review_comment":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pantsbuild/pants/pulls/5490/commits"},"statuses":{"href":"https://api.github.com/repos/pantsbuild/pants/statuses/41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"stuhood"}}