{"action":"created","comment":{"url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments/208680401","pull_request_review_id":144540664,"id":208680401,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwODY4MDQwMQ==","diff_hunk":"@@ -0,0 +1,93 @@\n+# dirs.rb\n+# creates hdfs dirs\n+# run this in an hbase jruby shell\n+\n+# usage: hbase shell dirs.rb [user|groups] dir_file quota_yml home_dir perms\n+# user ex. hbase shell dirs.rb user /tmp/user_dirs /tmp/user_quotas.yml /user 0700\n+# group ex. hbase shell dirs.rb groups /tmp/group_dirs /tmp/group_quotas.yml /groups 0770\n+\n+include Java\n+\n+require 'yaml'\n+\n+import org.apache.hadoop.conf.Configuration\n+import org.apache.hadoop.fs.FileSystem\n+import org.apache.hadoop.fs.Path\n+import org.apache.hadoop.fs.permission.FsPermission\n+import org.apache.hadoop.hdfs.protocol.HdfsConstants\n+\n+usage = \"hbase shell \"\\\n+  \"dirs.rb [user|groups] dir_file quota_yml home_dir perms\"\n+banner = \"usage: #{usage}\\n  -- need \"\n+\n+config = {\n+  'mode' => (ARGV[0] or raise \"#{banner} [user|groups]\"),\n+  'dir_file' => (ARGV[1] or raise \"#{banner} dir_file\"),\n+  'quota_file' => (ARGV[2] or raise \"#{banner} quota_yml\"),\n+  'home' => (ARGV[3] or raise \"#{banner} home_dir\"),\n+  'perms' => (ARGV[4] or raise \"#{banner} perms\"),\n+}\n+\n+# hdfs file system handle\n+fs = FileSystem.newInstance(Configuration.new)\n+\n+# create home directory\n+path = Path.new(config['home'])\n+fs.mkdirs(path)\n+fs.setOwner(path, 'hdfs', 'hdfs')\n+fs.setPermission(path, FsPermission.new(0775))\n+\n+# load quotas from file\n+quotas = YAML.load_file(config['quota_file'])\n+\n+File.open(config['dir_file']) do |f|\n+  f.each_line do |dir|\n+    dir = dir.strip\n+    path = Path.new(\"#{config['home']}/#{dir}\")\n+    puts \"updating path: #{path}\"\n+\n+    # enforce ownership and permissions\n+    owner = config['mode'] == 'user' ? dir : 'hdfs'\n+    group = config['mode'] == 'groups' ? dir : 'hdfs'\n+    perms = FsPermission.new(config['perms'].to_i(8))\n+\n+    # create the directory\n+    fs.mkdirs(path)\n+    fs.setOwner(path, owner, group)\n+    fs.setPermission(path, perms)\n+\n+    # Takes a string with a unit-prefix\n+    # returns the result as a \"long\"\n+    # Examples are:\n+    #   string2long(600) => 600\n+    #   string2long('1') => 1\n+    #   string2long('1k') => 1024\n+    #   string2long('30T') => 32985348833280\n+    def string2long(s)\n+      s = s.to_s\n+      prefixes = {\n+        ''  => 1,\n+        'k' => 1024,\n+        'm' => 1024**2,\n+        'g' => 1024**3,\n+        't' => 1024**4,\n+        'p' => 1024**5\n+      }\n+\n+      base = s.sub(/[kmgtp]/, '').to_i\n+      prefix = prefixes[s.sub(/[0-9.]*/,'').downcase]\n+\n+      return base * prefix\n+    end\n+\n+    # set space quota\n+    if quotas[dir]\n+      max_quota = HdfsConstants::QUOTA_DONT_SET","path":"cookbooks/bcpc-hadoop/files/default/dirs.rb","position":null,"original_position":85,"commit_id":"6cfe6f74c62c1f9967105ab4748ba5573475ef12","original_commit_id":"0cd3cc695a620b733c17a600dccff10b38272d75","user":{"login":"cbaenziger","id":4192761,"node_id":"MDQ6VXNlcjQxOTI3NjE=","avatar_url":"https://avatars2.githubusercontent.com/u/4192761?v=4","gravatar_id":"","url":"https://api.github.com/users/cbaenziger","html_url":"https://github.com/cbaenziger","followers_url":"https://api.github.com/users/cbaenziger/followers","following_url":"https://api.github.com/users/cbaenziger/following{/other_user}","gists_url":"https://api.github.com/users/cbaenziger/gists{/gist_id}","starred_url":"https://api.github.com/users/cbaenziger/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cbaenziger/subscriptions","organizations_url":"https://api.github.com/users/cbaenziger/orgs","repos_url":"https://api.github.com/users/cbaenziger/repos","events_url":"https://api.github.com/users/cbaenziger/events{/privacy}","received_events_url":"https://api.github.com/users/cbaenziger/received_events","type":"User","site_admin":false},"body":"Any reason to not use `HdfsConstants.QUOTA_RESET`?","created_at":"2018-08-08T18:05:34Z","updated_at":"2018-08-08T18:36:28Z","html_url":"https://github.com/bloomberg/chef-bach/pull/1254#discussion_r208680401","pull_request_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments/208680401"},"html":{"href":"https://github.com/bloomberg/chef-bach/pull/1254#discussion_r208680401"},"pull_request":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254"}}},"pull_request":{"url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254","id":207057061,"node_id":"MDExOlB1bGxSZXF1ZXN0MjA3MDU3MDYx","html_url":"https://github.com/bloomberg/chef-bach/pull/1254","diff_url":"https://github.com/bloomberg/chef-bach/pull/1254.diff","patch_url":"https://github.com/bloomberg/chef-bach/pull/1254.patch","issue_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/1254","number":1254,"state":"closed","locked":false,"title":"fix ldapsearch for hdfs dir creation","user":{"login":"ronny-macmaster","id":11374244,"node_id":"MDQ6VXNlcjExMzc0MjQ0","avatar_url":"https://avatars0.githubusercontent.com/u/11374244?v=4","gravatar_id":"","url":"https://api.github.com/users/ronny-macmaster","html_url":"https://github.com/ronny-macmaster","followers_url":"https://api.github.com/users/ronny-macmaster/followers","following_url":"https://api.github.com/users/ronny-macmaster/following{/other_user}","gists_url":"https://api.github.com/users/ronny-macmaster/gists{/gist_id}","starred_url":"https://api.github.com/users/ronny-macmaster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ronny-macmaster/subscriptions","organizations_url":"https://api.github.com/users/ronny-macmaster/orgs","repos_url":"https://api.github.com/users/ronny-macmaster/repos","events_url":"https://api.github.com/users/ronny-macmaster/events{/privacy}","received_events_url":"https://api.github.com/users/ronny-macmaster/received_events","type":"User","site_admin":false},"body":"These changes should allow hdfs user and group directory provisioning to work in bcpc-hadoop with ldapsearch commands. Some hdfs attributes have been refactored to logically group them in an ldap.rb attributes file. I have tested these changes on a hardware cluster.","created_at":"2018-08-08T15:57:43Z","updated_at":"2018-08-08T18:36:28Z","closed_at":"2018-08-08T18:28:05Z","merged_at":"2018-08-08T18:28:05Z","merge_commit_sha":"7531d3ac0a46553a80394b4357cd9e28d5d6b668","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":135806804,"node_id":"MDU6TGFiZWwxMzU4MDY4MDQ=","url":"https://api.github.com/repos/bloomberg/chef-bach/labels/Enhancement","name":"Enhancement","color":"cccccc","default":false},{"id":344072888,"node_id":"MDU6TGFiZWwzNDQwNzI4ODg=","url":"https://api.github.com/repos/bloomberg/chef-bach/labels/Tested","name":"Tested","color":"009800","default":false},{"id":777014557,"node_id":"MDU6TGFiZWw3NzcwMTQ1NTc=","url":"https://api.github.com/repos/bloomberg/chef-bach/labels/to-backport","name":"to-backport","color":"fbca04","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254/commits","review_comments_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254/comments","review_comment_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/1254/comments","statuses_url":"https://api.github.com/repos/bloomberg/chef-bach/statuses/6cfe6f74c62c1f9967105ab4748ba5573475ef12","head":{"label":"ronny-macmaster:fix-dir-creation","ref":"fix-dir-creation","sha":"6cfe6f74c62c1f9967105ab4748ba5573475ef12","user":{"login":"ronny-macmaster","id":11374244,"node_id":"MDQ6VXNlcjExMzc0MjQ0","avatar_url":"https://avatars0.githubusercontent.com/u/11374244?v=4","gravatar_id":"","url":"https://api.github.com/users/ronny-macmaster","html_url":"https://github.com/ronny-macmaster","followers_url":"https://api.github.com/users/ronny-macmaster/followers","following_url":"https://api.github.com/users/ronny-macmaster/following{/other_user}","gists_url":"https://api.github.com/users/ronny-macmaster/gists{/gist_id}","starred_url":"https://api.github.com/users/ronny-macmaster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ronny-macmaster/subscriptions","organizations_url":"https://api.github.com/users/ronny-macmaster/orgs","repos_url":"https://api.github.com/users/ronny-macmaster/repos","events_url":"https://api.github.com/users/ronny-macmaster/events{/privacy}","received_events_url":"https://api.github.com/users/ronny-macmaster/received_events","type":"User","site_admin":false},"repo":{"id":119592090,"node_id":"MDEwOlJlcG9zaXRvcnkxMTk1OTIwOTA=","name":"chef-bach","full_name":"ronny-macmaster/chef-bach","owner":{"login":"ronny-macmaster","id":11374244,"node_id":"MDQ6VXNlcjExMzc0MjQ0","avatar_url":"https://avatars0.githubusercontent.com/u/11374244?v=4","gravatar_id":"","url":"https://api.github.com/users/ronny-macmaster","html_url":"https://github.com/ronny-macmaster","followers_url":"https://api.github.com/users/ronny-macmaster/followers","following_url":"https://api.github.com/users/ronny-macmaster/following{/other_user}","gists_url":"https://api.github.com/users/ronny-macmaster/gists{/gist_id}","starred_url":"https://api.github.com/users/ronny-macmaster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ronny-macmaster/subscriptions","organizations_url":"https://api.github.com/users/ronny-macmaster/orgs","repos_url":"https://api.github.com/users/ronny-macmaster/repos","events_url":"https://api.github.com/users/ronny-macmaster/events{/privacy}","received_events_url":"https://api.github.com/users/ronny-macmaster/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ronny-macmaster/chef-bach","description":"Chef recipes for Bloomberg's deployment of Hadoop and related components","fork":true,"url":"https://api.github.com/repos/ronny-macmaster/chef-bach","forks_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/forks","keys_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/teams","hooks_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/hooks","issue_events_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/issues/events{/number}","events_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/events","assignees_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/assignees{/user}","branches_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/branches{/branch}","tags_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/tags","blobs_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/git/refs{/sha}","trees_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/statuses/{sha}","languages_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/languages","stargazers_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/stargazers","contributors_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/contributors","subscribers_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/subscribers","subscription_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/subscription","commits_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/commits{/sha}","git_commits_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/git/commits{/sha}","comments_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/comments{/number}","issue_comment_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/issues/comments{/number}","contents_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/contents/{+path}","compare_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/merges","archive_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/downloads","issues_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/issues{/number}","pulls_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/pulls{/number}","milestones_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/milestones{/number}","notifications_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/labels{/name}","releases_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/releases{/id}","deployments_url":"https://api.github.com/repos/ronny-macmaster/chef-bach/deployments","created_at":"2018-01-30T20:44:17Z","updated_at":"2018-08-03T17:20:37Z","pushed_at":"2018-08-08T18:17:43Z","git_url":"git://github.com/ronny-macmaster/chef-bach.git","ssh_url":"git@github.com:ronny-macmaster/chef-bach.git","clone_url":"https://github.com/ronny-macmaster/chef-bach.git","svn_url":"https://github.com/ronny-macmaster/chef-bach","homepage":null,"size":6733,"stargazers_count":0,"watchers_count":0,"language":"Ruby","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"bloomberg:master","ref":"master","sha":"e3bf474d70e0fe2e16a5b6ea29563dde1ef18276","user":{"login":"bloomberg","id":1416818,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTY4MTg=","avatar_url":"https://avatars2.githubusercontent.com/u/1416818?v=4","gravatar_id":"","url":"https://api.github.com/users/bloomberg","html_url":"https://github.com/bloomberg","followers_url":"https://api.github.com/users/bloomberg/followers","following_url":"https://api.github.com/users/bloomberg/following{/other_user}","gists_url":"https://api.github.com/users/bloomberg/gists{/gist_id}","starred_url":"https://api.github.com/users/bloomberg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bloomberg/subscriptions","organizations_url":"https://api.github.com/users/bloomberg/orgs","repos_url":"https://api.github.com/users/bloomberg/repos","events_url":"https://api.github.com/users/bloomberg/events{/privacy}","received_events_url":"https://api.github.com/users/bloomberg/received_events","type":"Organization","site_admin":false},"repo":{"id":24933148,"node_id":"MDEwOlJlcG9zaXRvcnkyNDkzMzE0OA==","name":"chef-bach","full_name":"bloomberg/chef-bach","owner":{"login":"bloomberg","id":1416818,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTY4MTg=","avatar_url":"https://avatars2.githubusercontent.com/u/1416818?v=4","gravatar_id":"","url":"https://api.github.com/users/bloomberg","html_url":"https://github.com/bloomberg","followers_url":"https://api.github.com/users/bloomberg/followers","following_url":"https://api.github.com/users/bloomberg/following{/other_user}","gists_url":"https://api.github.com/users/bloomberg/gists{/gist_id}","starred_url":"https://api.github.com/users/bloomberg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bloomberg/subscriptions","organizations_url":"https://api.github.com/users/bloomberg/orgs","repos_url":"https://api.github.com/users/bloomberg/repos","events_url":"https://api.github.com/users/bloomberg/events{/privacy}","received_events_url":"https://api.github.com/users/bloomberg/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/bloomberg/chef-bach","description":"Chef recipes for Bloomberg's deployment of Hadoop and related components","fork":false,"url":"https://api.github.com/repos/bloomberg/chef-bach","forks_url":"https://api.github.com/repos/bloomberg/chef-bach/forks","keys_url":"https://api.github.com/repos/bloomberg/chef-bach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bloomberg/chef-bach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bloomberg/chef-bach/teams","hooks_url":"https://api.github.com/repos/bloomberg/chef-bach/hooks","issue_events_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/events{/number}","events_url":"https://api.github.com/repos/bloomberg/chef-bach/events","assignees_url":"https://api.github.com/repos/bloomberg/chef-bach/assignees{/user}","branches_url":"https://api.github.com/repos/bloomberg/chef-bach/branches{/branch}","tags_url":"https://api.github.com/repos/bloomberg/chef-bach/tags","blobs_url":"https://api.github.com/repos/bloomberg/chef-bach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bloomberg/chef-bach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bloomberg/chef-bach/git/refs{/sha}","trees_url":"https://api.github.com/repos/bloomberg/chef-bach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bloomberg/chef-bach/statuses/{sha}","languages_url":"https://api.github.com/repos/bloomberg/chef-bach/languages","stargazers_url":"https://api.github.com/repos/bloomberg/chef-bach/stargazers","contributors_url":"https://api.github.com/repos/bloomberg/chef-bach/contributors","subscribers_url":"https://api.github.com/repos/bloomberg/chef-bach/subscribers","subscription_url":"https://api.github.com/repos/bloomberg/chef-bach/subscription","commits_url":"https://api.github.com/repos/bloomberg/chef-bach/commits{/sha}","git_commits_url":"https://api.github.com/repos/bloomberg/chef-bach/git/commits{/sha}","comments_url":"https://api.github.com/repos/bloomberg/chef-bach/comments{/number}","issue_comment_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/comments{/number}","contents_url":"https://api.github.com/repos/bloomberg/chef-bach/contents/{+path}","compare_url":"https://api.github.com/repos/bloomberg/chef-bach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bloomberg/chef-bach/merges","archive_url":"https://api.github.com/repos/bloomberg/chef-bach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bloomberg/chef-bach/downloads","issues_url":"https://api.github.com/repos/bloomberg/chef-bach/issues{/number}","pulls_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls{/number}","milestones_url":"https://api.github.com/repos/bloomberg/chef-bach/milestones{/number}","notifications_url":"https://api.github.com/repos/bloomberg/chef-bach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bloomberg/chef-bach/labels{/name}","releases_url":"https://api.github.com/repos/bloomberg/chef-bach/releases{/id}","deployments_url":"https://api.github.com/repos/bloomberg/chef-bach/deployments","created_at":"2014-10-08T09:16:26Z","updated_at":"2018-08-08T18:28:09Z","pushed_at":"2018-08-08T18:28:06Z","git_url":"git://github.com/bloomberg/chef-bach.git","ssh_url":"git@github.com:bloomberg/chef-bach.git","clone_url":"https://github.com/bloomberg/chef-bach.git","svn_url":"https://github.com/bloomberg/chef-bach","homepage":null,"size":6687,"stargazers_count":59,"watchers_count":59,"language":"Ruby","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":57,"mirror_url":null,"archived":false,"open_issues_count":135,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":57,"open_issues":135,"watchers":59,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254"},"html":{"href":"https://github.com/bloomberg/chef-bach/pull/1254"},"issue":{"href":"https://api.github.com/repos/bloomberg/chef-bach/issues/1254"},"comments":{"href":"https://api.github.com/repos/bloomberg/chef-bach/issues/1254/comments"},"review_comments":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254/comments"},"review_comment":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/1254/commits"},"statuses":{"href":"https://api.github.com/repos/bloomberg/chef-bach/statuses/6cfe6f74c62c1f9967105ab4748ba5573475ef12"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"cbaenziger"}}