{"action":"closed","number":1995,"pull_request":{"url":"https://api.github.com/repos/prestodb/presto/pulls/1995","id":24716184,"html_url":"https://github.com/prestodb/presto/pull/1995","diff_url":"https://github.com/prestodb/presto/pull/1995.diff","patch_url":"https://github.com/prestodb/presto/pull/1995.patch","issue_url":"https://api.github.com/repos/prestodb/presto/issues/1995","number":1995,"state":"closed","locked":false,"title":"Batch partition loading from the metastore","user":{"login":"nezihyigitbasi","id":1223839,"avatar_url":"https://avatars.githubusercontent.com/u/1223839?v=3","gravatar_id":"","url":"https://api.github.com/users/nezihyigitbasi","html_url":"https://github.com/nezihyigitbasi","followers_url":"https://api.github.com/users/nezihyigitbasi/followers","following_url":"https://api.github.com/users/nezihyigitbasi/following{/other_user}","gists_url":"https://api.github.com/users/nezihyigitbasi/gists{/gist_id}","starred_url":"https://api.github.com/users/nezihyigitbasi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nezihyigitbasi/subscriptions","organizations_url":"https://api.github.com/users/nezihyigitbasi/orgs","repos_url":"https://api.github.com/users/nezihyigitbasi/repos","events_url":"https://api.github.com/users/nezihyigitbasi/events{/privacy}","received_events_url":"https://api.github.com/users/nezihyigitbasi/received_events","type":"User","site_admin":false},"body":"OK here is the initial implementation that does batching based on time (a new parameter ``hive.metastore-partition-load-batch-interval`` is introduced) to address #1981 . Currently each hive-metastore thread does batching independently (instead of using locking and letting one thread do the batching as the same ``BackgroundCacheLoader`` is shared by all metastore threads). For keeping the state this implementation uses thread local variables to get rid of synchronization & make the code more concise. Once the code is ready to merge I will update the docs with the new parameter.","created_at":"2014-11-19T19:57:41Z","updated_at":"2016-04-29T23:21:50Z","closed_at":"2016-04-29T23:21:50Z","merged_at":null,"merge_commit_sha":"27ec98101e5055dbae6a34e300450a521a914435","assignee":{"login":"cberner","id":842688,"avatar_url":"https://avatars.githubusercontent.com/u/842688?v=3","gravatar_id":"","url":"https://api.github.com/users/cberner","html_url":"https://github.com/cberner","followers_url":"https://api.github.com/users/cberner/followers","following_url":"https://api.github.com/users/cberner/following{/other_user}","gists_url":"https://api.github.com/users/cberner/gists{/gist_id}","starred_url":"https://api.github.com/users/cberner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cberner/subscriptions","organizations_url":"https://api.github.com/users/cberner/orgs","repos_url":"https://api.github.com/users/cberner/repos","events_url":"https://api.github.com/users/cberner/events{/privacy}","received_events_url":"https://api.github.com/users/cberner/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/prestodb/presto/pulls/1995/commits","review_comments_url":"https://api.github.com/repos/prestodb/presto/pulls/1995/comments","review_comment_url":"https://api.github.com/repos/prestodb/presto/pulls/comments{/number}","comments_url":"https://api.github.com/repos/prestodb/presto/issues/1995/comments","statuses_url":"https://api.github.com/repos/prestodb/presto/statuses/b79dc4e408f3e12f653790ca098ba6e8fbd9e81e","head":{"label":"nezihyigitbasi:hive_metastore_batch","ref":"hive_metastore_batch","sha":"b79dc4e408f3e12f653790ca098ba6e8fbd9e81e","user":{"login":"nezihyigitbasi","id":1223839,"avatar_url":"https://avatars.githubusercontent.com/u/1223839?v=3","gravatar_id":"","url":"https://api.github.com/users/nezihyigitbasi","html_url":"https://github.com/nezihyigitbasi","followers_url":"https://api.github.com/users/nezihyigitbasi/followers","following_url":"https://api.github.com/users/nezihyigitbasi/following{/other_user}","gists_url":"https://api.github.com/users/nezihyigitbasi/gists{/gist_id}","starred_url":"https://api.github.com/users/nezihyigitbasi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nezihyigitbasi/subscriptions","organizations_url":"https://api.github.com/users/nezihyigitbasi/orgs","repos_url":"https://api.github.com/users/nezihyigitbasi/repos","events_url":"https://api.github.com/users/nezihyigitbasi/events{/privacy}","received_events_url":"https://api.github.com/users/nezihyigitbasi/received_events","type":"User","site_admin":false},"repo":{"id":20698188,"name":"presto","full_name":"nezihyigitbasi/presto","owner":{"login":"nezihyigitbasi","id":1223839,"avatar_url":"https://avatars.githubusercontent.com/u/1223839?v=3","gravatar_id":"","url":"https://api.github.com/users/nezihyigitbasi","html_url":"https://github.com/nezihyigitbasi","followers_url":"https://api.github.com/users/nezihyigitbasi/followers","following_url":"https://api.github.com/users/nezihyigitbasi/following{/other_user}","gists_url":"https://api.github.com/users/nezihyigitbasi/gists{/gist_id}","starred_url":"https://api.github.com/users/nezihyigitbasi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nezihyigitbasi/subscriptions","organizations_url":"https://api.github.com/users/nezihyigitbasi/orgs","repos_url":"https://api.github.com/users/nezihyigitbasi/repos","events_url":"https://api.github.com/users/nezihyigitbasi/events{/privacy}","received_events_url":"https://api.github.com/users/nezihyigitbasi/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nezihyigitbasi/presto","description":"Distributed SQL query engine for big data","fork":true,"url":"https://api.github.com/repos/nezihyigitbasi/presto","forks_url":"https://api.github.com/repos/nezihyigitbasi/presto/forks","keys_url":"https://api.github.com/repos/nezihyigitbasi/presto/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nezihyigitbasi/presto/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nezihyigitbasi/presto/teams","hooks_url":"https://api.github.com/repos/nezihyigitbasi/presto/hooks","issue_events_url":"https://api.github.com/repos/nezihyigitbasi/presto/issues/events{/number}","events_url":"https://api.github.com/repos/nezihyigitbasi/presto/events","assignees_url":"https://api.github.com/repos/nezihyigitbasi/presto/assignees{/user}","branches_url":"https://api.github.com/repos/nezihyigitbasi/presto/branches{/branch}","tags_url":"https://api.github.com/repos/nezihyigitbasi/presto/tags","blobs_url":"https://api.github.com/repos/nezihyigitbasi/presto/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nezihyigitbasi/presto/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nezihyigitbasi/presto/git/refs{/sha}","trees_url":"https://api.github.com/repos/nezihyigitbasi/presto/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nezihyigitbasi/presto/statuses/{sha}","languages_url":"https://api.github.com/repos/nezihyigitbasi/presto/languages","stargazers_url":"https://api.github.com/repos/nezihyigitbasi/presto/stargazers","contributors_url":"https://api.github.com/repos/nezihyigitbasi/presto/contributors","subscribers_url":"https://api.github.com/repos/nezihyigitbasi/presto/subscribers","subscription_url":"https://api.github.com/repos/nezihyigitbasi/presto/subscription","commits_url":"https://api.github.com/repos/nezihyigitbasi/presto/commits{/sha}","git_commits_url":"https://api.github.com/repos/nezihyigitbasi/presto/git/commits{/sha}","comments_url":"https://api.github.com/repos/nezihyigitbasi/presto/comments{/number}","issue_comment_url":"https://api.github.com/repos/nezihyigitbasi/presto/issues/comments{/number}","contents_url":"https://api.github.com/repos/nezihyigitbasi/presto/contents/{+path}","compare_url":"https://api.github.com/repos/nezihyigitbasi/presto/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nezihyigitbasi/presto/merges","archive_url":"https://api.github.com/repos/nezihyigitbasi/presto/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nezihyigitbasi/presto/downloads","issues_url":"https://api.github.com/repos/nezihyigitbasi/presto/issues{/number}","pulls_url":"https://api.github.com/repos/nezihyigitbasi/presto/pulls{/number}","milestones_url":"https://api.github.com/repos/nezihyigitbasi/presto/milestones{/number}","notifications_url":"https://api.github.com/repos/nezihyigitbasi/presto/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nezihyigitbasi/presto/labels{/name}","releases_url":"https://api.github.com/repos/nezihyigitbasi/presto/releases{/id}","deployments_url":"https://api.github.com/repos/nezihyigitbasi/presto/deployments","created_at":"2014-06-10T19:14:21Z","updated_at":"2016-01-15T21:48:09Z","pushed_at":"2016-04-29T16:21:56Z","git_url":"git://github.com/nezihyigitbasi/presto.git","ssh_url":"git@github.com:nezihyigitbasi/presto.git","clone_url":"https://github.com/nezihyigitbasi/presto.git","svn_url":"https://github.com/nezihyigitbasi/presto","homepage":"http://prestodb.io/","size":52447,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"prestodb:master","ref":"master","sha":"8abfad15187059766922f76e22cc7109a2c6e546","user":{"login":"prestodb","id":6882181,"avatar_url":"https://avatars.githubusercontent.com/u/6882181?v=3","gravatar_id":"","url":"https://api.github.com/users/prestodb","html_url":"https://github.com/prestodb","followers_url":"https://api.github.com/users/prestodb/followers","following_url":"https://api.github.com/users/prestodb/following{/other_user}","gists_url":"https://api.github.com/users/prestodb/gists{/gist_id}","starred_url":"https://api.github.com/users/prestodb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prestodb/subscriptions","organizations_url":"https://api.github.com/users/prestodb/orgs","repos_url":"https://api.github.com/users/prestodb/repos","events_url":"https://api.github.com/users/prestodb/events{/privacy}","received_events_url":"https://api.github.com/users/prestodb/received_events","type":"Organization","site_admin":false},"repo":{"id":5349565,"name":"presto","full_name":"prestodb/presto","owner":{"login":"prestodb","id":6882181,"avatar_url":"https://avatars.githubusercontent.com/u/6882181?v=3","gravatar_id":"","url":"https://api.github.com/users/prestodb","html_url":"https://github.com/prestodb","followers_url":"https://api.github.com/users/prestodb/followers","following_url":"https://api.github.com/users/prestodb/following{/other_user}","gists_url":"https://api.github.com/users/prestodb/gists{/gist_id}","starred_url":"https://api.github.com/users/prestodb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prestodb/subscriptions","organizations_url":"https://api.github.com/users/prestodb/orgs","repos_url":"https://api.github.com/users/prestodb/repos","events_url":"https://api.github.com/users/prestodb/events{/privacy}","received_events_url":"https://api.github.com/users/prestodb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/prestodb/presto","description":"Distributed SQL query engine for big data","fork":false,"url":"https://api.github.com/repos/prestodb/presto","forks_url":"https://api.github.com/repos/prestodb/presto/forks","keys_url":"https://api.github.com/repos/prestodb/presto/keys{/key_id}","collaborators_url":"https://api.github.com/repos/prestodb/presto/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/prestodb/presto/teams","hooks_url":"https://api.github.com/repos/prestodb/presto/hooks","issue_events_url":"https://api.github.com/repos/prestodb/presto/issues/events{/number}","events_url":"https://api.github.com/repos/prestodb/presto/events","assignees_url":"https://api.github.com/repos/prestodb/presto/assignees{/user}","branches_url":"https://api.github.com/repos/prestodb/presto/branches{/branch}","tags_url":"https://api.github.com/repos/prestodb/presto/tags","blobs_url":"https://api.github.com/repos/prestodb/presto/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/prestodb/presto/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/prestodb/presto/git/refs{/sha}","trees_url":"https://api.github.com/repos/prestodb/presto/git/trees{/sha}","statuses_url":"https://api.github.com/repos/prestodb/presto/statuses/{sha}","languages_url":"https://api.github.com/repos/prestodb/presto/languages","stargazers_url":"https://api.github.com/repos/prestodb/presto/stargazers","contributors_url":"https://api.github.com/repos/prestodb/presto/contributors","subscribers_url":"https://api.github.com/repos/prestodb/presto/subscribers","subscription_url":"https://api.github.com/repos/prestodb/presto/subscription","commits_url":"https://api.github.com/repos/prestodb/presto/commits{/sha}","git_commits_url":"https://api.github.com/repos/prestodb/presto/git/commits{/sha}","comments_url":"https://api.github.com/repos/prestodb/presto/comments{/number}","issue_comment_url":"https://api.github.com/repos/prestodb/presto/issues/comments{/number}","contents_url":"https://api.github.com/repos/prestodb/presto/contents/{+path}","compare_url":"https://api.github.com/repos/prestodb/presto/compare/{base}...{head}","merges_url":"https://api.github.com/repos/prestodb/presto/merges","archive_url":"https://api.github.com/repos/prestodb/presto/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/prestodb/presto/downloads","issues_url":"https://api.github.com/repos/prestodb/presto/issues{/number}","pulls_url":"https://api.github.com/repos/prestodb/presto/pulls{/number}","milestones_url":"https://api.github.com/repos/prestodb/presto/milestones{/number}","notifications_url":"https://api.github.com/repos/prestodb/presto/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/prestodb/presto/labels{/name}","releases_url":"https://api.github.com/repos/prestodb/presto/releases{/id}","deployments_url":"https://api.github.com/repos/prestodb/presto/deployments","created_at":"2012-08-09T01:03:37Z","updated_at":"2016-04-29T22:49:57Z","pushed_at":"2016-04-29T23:18:39Z","git_url":"git://github.com/prestodb/presto.git","ssh_url":"git@github.com:prestodb/presto.git","clone_url":"https://github.com/prestodb/presto.git","svn_url":"https://github.com/prestodb/presto","homepage":"https://prestodb.io","size":150062,"stargazers_count":4706,"watchers_count":4706,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1368,"mirror_url":null,"open_issues_count":557,"forks":1368,"open_issues":557,"watchers":4706,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/prestodb/presto/pulls/1995"},"html":{"href":"https://github.com/prestodb/presto/pull/1995"},"issue":{"href":"https://api.github.com/repos/prestodb/presto/issues/1995"},"comments":{"href":"https://api.github.com/repos/prestodb/presto/issues/1995/comments"},"review_comments":{"href":"https://api.github.com/repos/prestodb/presto/pulls/1995/comments"},"review_comment":{"href":"https://api.github.com/repos/prestodb/presto/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/prestodb/presto/pulls/1995/commits"},"statuses":{"href":"https://api.github.com/repos/prestodb/presto/statuses/b79dc4e408f3e12f653790ca098ba6e8fbd9e81e"}},"merged":false,"mergeable":false,"mergeable_state":"dirty","merged_by":null,"comments":9,"review_comments":16,"commits":1,"additions":229,"deletions":45,"changed_files":8}}