{"action":"created","comment":{"url":"https://api.github.com/repos/JuliaLang/julia/pulls/comments/76985574","id":76985574,"diff_hunk":"@@ -28,7 +43,38 @@ function available(names=readdir(\"METADATA\"))\n     end\n     return pkgs\n end\n-available(pkg::AbstractString) = get(available([pkg]),pkg,Dict{VersionNumber,Available}())\n+available(pkg::AbstractString) = get(_available([pkg]),pkg,Dict{VersionNumber,Available}())\n+\n+# Uses cached data if not outdated\n+function available(cache::AvailableCache = PKG_AVAILABLE_CACHE)\n+    names = readdir(\"METADATA\")\n+    # Not a git repo so just bail on using cache\n+    if in(\".git\", names)\n+        pkg, usecache = LibGit2.with(LibGit2.GitRepo(\"METADATA\")) do repo\n+            LibGit2.with(LibGit2.head(repo)) do head\n+                sha = string(Base.LibGit2.Oid(head))","path":"base/pkg/read.jl","position":38,"original_position":38,"commit_id":"77e8789c2d528c30c504564d670f2328fd72e884","original_commit_id":"77e8789c2d528c30c504564d670f2328fd72e884","user":{"login":"tkelman","id":5934628,"avatar_url":"https://avatars.githubusercontent.com/u/5934628?v=3","gravatar_id":"","url":"https://api.github.com/users/tkelman","html_url":"https://github.com/tkelman","followers_url":"https://api.github.com/users/tkelman/followers","following_url":"https://api.github.com/users/tkelman/following{/other_user}","gists_url":"https://api.github.com/users/tkelman/gists{/gist_id}","starred_url":"https://api.github.com/users/tkelman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tkelman/subscriptions","organizations_url":"https://api.github.com/users/tkelman/orgs","repos_url":"https://api.github.com/users/tkelman/repos","events_url":"https://api.github.com/users/tkelman/events{/privacy}","received_events_url":"https://api.github.com/users/tkelman/received_events","type":"User","site_admin":false},"body":"worth storing this as bytes instead of a string?","created_at":"2016-08-31T13:22:22Z","updated_at":"2016-08-31T13:22:22Z","html_url":"https://github.com/JuliaLang/julia/pull/18289#discussion_r76985574","pull_request_url":"https://api.github.com/repos/JuliaLang/julia/pulls/18289","_links":{"self":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/comments/76985574"},"html":{"href":"https://github.com/JuliaLang/julia/pull/18289#discussion_r76985574"},"pull_request":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/18289"}}},"pull_request":{"url":"https://api.github.com/repos/JuliaLang/julia/pulls/18289","id":83225782,"html_url":"https://github.com/JuliaLang/julia/pull/18289","diff_url":"https://github.com/JuliaLang/julia/pull/18289.diff","patch_url":"https://github.com/JuliaLang/julia/pull/18289.patch","issue_url":"https://api.github.com/repos/JuliaLang/julia/issues/18289","number":18289,"state":"open","locked":false,"title":"RFC: add cache for Pkg.Read.available","user":{"login":"KristofferC","id":1282691,"avatar_url":"https://avatars.githubusercontent.com/u/1282691?v=3","gravatar_id":"","url":"https://api.github.com/users/KristofferC","html_url":"https://github.com/KristofferC","followers_url":"https://api.github.com/users/KristofferC/followers","following_url":"https://api.github.com/users/KristofferC/following{/other_user}","gists_url":"https://api.github.com/users/KristofferC/gists{/gist_id}","starred_url":"https://api.github.com/users/KristofferC/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/KristofferC/subscriptions","organizations_url":"https://api.github.com/users/KristofferC/orgs","repos_url":"https://api.github.com/users/KristofferC/repos","events_url":"https://api.github.com/users/KristofferC/events{/privacy}","received_events_url":"https://api.github.com/users/KristofferC/received_events","type":"User","site_admin":false},"body":"Many of the functions in `Pkg` call out to `Pkg.Read.available()` which searches though all of METADATA and reparses all the files. This PR introduces a cache so that in case of the METADATA being in the same state as the previous call, the previous result is then used.\r\n\r\nUnfortunately, for some reason (probably the way we structure METADATA with a huge number of files) just runnning the equivalent of `git status` in libgit2 takes ~ 0.1 - 0.3 seconds depending on computer so  even when using the cache it takes a bit of time. FWIW, the `status` command on https://github.com/KristofferC/METADATA_compressed takes 0.005 seconds.\r\n\r\nI have more optimizations for Pkg stuff in the pipeline but let's take them one by one.\r\n","created_at":"2016-08-30T08:08:39Z","updated_at":"2016-08-31T13:22:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"59227d716b7749b37e973f7ce91751a253821c5b","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/JuliaLang/julia/pulls/18289/commits","review_comments_url":"https://api.github.com/repos/JuliaLang/julia/pulls/18289/comments","review_comment_url":"https://api.github.com/repos/JuliaLang/julia/pulls/comments{/number}","comments_url":"https://api.github.com/repos/JuliaLang/julia/issues/18289/comments","statuses_url":"https://api.github.com/repos/JuliaLang/julia/statuses/77e8789c2d528c30c504564d670f2328fd72e884","head":{"label":"JuliaLang:kc/cache_available","ref":"kc/cache_available","sha":"77e8789c2d528c30c504564d670f2328fd72e884","user":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"repo":{"id":1644196,"name":"julia","full_name":"JuliaLang/julia","owner":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/JuliaLang/julia","description":"The Julia Language: A fresh approach to technical computing.","fork":false,"url":"https://api.github.com/repos/JuliaLang/julia","forks_url":"https://api.github.com/repos/JuliaLang/julia/forks","keys_url":"https://api.github.com/repos/JuliaLang/julia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JuliaLang/julia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JuliaLang/julia/teams","hooks_url":"https://api.github.com/repos/JuliaLang/julia/hooks","issue_events_url":"https://api.github.com/repos/JuliaLang/julia/issues/events{/number}","events_url":"https://api.github.com/repos/JuliaLang/julia/events","assignees_url":"https://api.github.com/repos/JuliaLang/julia/assignees{/user}","branches_url":"https://api.github.com/repos/JuliaLang/julia/branches{/branch}","tags_url":"https://api.github.com/repos/JuliaLang/julia/tags","blobs_url":"https://api.github.com/repos/JuliaLang/julia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JuliaLang/julia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JuliaLang/julia/git/refs{/sha}","trees_url":"https://api.github.com/repos/JuliaLang/julia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JuliaLang/julia/statuses/{sha}","languages_url":"https://api.github.com/repos/JuliaLang/julia/languages","stargazers_url":"https://api.github.com/repos/JuliaLang/julia/stargazers","contributors_url":"https://api.github.com/repos/JuliaLang/julia/contributors","subscribers_url":"https://api.github.com/repos/JuliaLang/julia/subscribers","subscription_url":"https://api.github.com/repos/JuliaLang/julia/subscription","commits_url":"https://api.github.com/repos/JuliaLang/julia/commits{/sha}","git_commits_url":"https://api.github.com/repos/JuliaLang/julia/git/commits{/sha}","comments_url":"https://api.github.com/repos/JuliaLang/julia/comments{/number}","issue_comment_url":"https://api.github.com/repos/JuliaLang/julia/issues/comments{/number}","contents_url":"https://api.github.com/repos/JuliaLang/julia/contents/{+path}","compare_url":"https://api.github.com/repos/JuliaLang/julia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JuliaLang/julia/merges","archive_url":"https://api.github.com/repos/JuliaLang/julia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JuliaLang/julia/downloads","issues_url":"https://api.github.com/repos/JuliaLang/julia/issues{/number}","pulls_url":"https://api.github.com/repos/JuliaLang/julia/pulls{/number}","milestones_url":"https://api.github.com/repos/JuliaLang/julia/milestones{/number}","notifications_url":"https://api.github.com/repos/JuliaLang/julia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JuliaLang/julia/labels{/name}","releases_url":"https://api.github.com/repos/JuliaLang/julia/releases{/id}","deployments_url":"https://api.github.com/repos/JuliaLang/julia/deployments","created_at":"2011-04-21T07:01:50Z","updated_at":"2016-08-31T11:05:06Z","pushed_at":"2016-08-31T12:41:14Z","git_url":"git://github.com/JuliaLang/julia.git","ssh_url":"git@github.com:JuliaLang/julia.git","clone_url":"https://github.com/JuliaLang/julia.git","svn_url":"https://github.com/JuliaLang/julia","homepage":"http://julialang.org/","size":411607,"stargazers_count":7441,"watchers_count":7441,"language":"Julia","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1737,"mirror_url":null,"open_issues_count":1941,"forks":1737,"open_issues":1941,"watchers":7441,"default_branch":"master"}},"base":{"label":"JuliaLang:master","ref":"master","sha":"cc30dd151e80494d7b52250ecdc13af917dd711c","user":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"repo":{"id":1644196,"name":"julia","full_name":"JuliaLang/julia","owner":{"login":"JuliaLang","id":743164,"avatar_url":"https://avatars.githubusercontent.com/u/743164?v=3","gravatar_id":"","url":"https://api.github.com/users/JuliaLang","html_url":"https://github.com/JuliaLang","followers_url":"https://api.github.com/users/JuliaLang/followers","following_url":"https://api.github.com/users/JuliaLang/following{/other_user}","gists_url":"https://api.github.com/users/JuliaLang/gists{/gist_id}","starred_url":"https://api.github.com/users/JuliaLang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JuliaLang/subscriptions","organizations_url":"https://api.github.com/users/JuliaLang/orgs","repos_url":"https://api.github.com/users/JuliaLang/repos","events_url":"https://api.github.com/users/JuliaLang/events{/privacy}","received_events_url":"https://api.github.com/users/JuliaLang/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/JuliaLang/julia","description":"The Julia Language: A fresh approach to technical computing.","fork":false,"url":"https://api.github.com/repos/JuliaLang/julia","forks_url":"https://api.github.com/repos/JuliaLang/julia/forks","keys_url":"https://api.github.com/repos/JuliaLang/julia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JuliaLang/julia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JuliaLang/julia/teams","hooks_url":"https://api.github.com/repos/JuliaLang/julia/hooks","issue_events_url":"https://api.github.com/repos/JuliaLang/julia/issues/events{/number}","events_url":"https://api.github.com/repos/JuliaLang/julia/events","assignees_url":"https://api.github.com/repos/JuliaLang/julia/assignees{/user}","branches_url":"https://api.github.com/repos/JuliaLang/julia/branches{/branch}","tags_url":"https://api.github.com/repos/JuliaLang/julia/tags","blobs_url":"https://api.github.com/repos/JuliaLang/julia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JuliaLang/julia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JuliaLang/julia/git/refs{/sha}","trees_url":"https://api.github.com/repos/JuliaLang/julia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JuliaLang/julia/statuses/{sha}","languages_url":"https://api.github.com/repos/JuliaLang/julia/languages","stargazers_url":"https://api.github.com/repos/JuliaLang/julia/stargazers","contributors_url":"https://api.github.com/repos/JuliaLang/julia/contributors","subscribers_url":"https://api.github.com/repos/JuliaLang/julia/subscribers","subscription_url":"https://api.github.com/repos/JuliaLang/julia/subscription","commits_url":"https://api.github.com/repos/JuliaLang/julia/commits{/sha}","git_commits_url":"https://api.github.com/repos/JuliaLang/julia/git/commits{/sha}","comments_url":"https://api.github.com/repos/JuliaLang/julia/comments{/number}","issue_comment_url":"https://api.github.com/repos/JuliaLang/julia/issues/comments{/number}","contents_url":"https://api.github.com/repos/JuliaLang/julia/contents/{+path}","compare_url":"https://api.github.com/repos/JuliaLang/julia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JuliaLang/julia/merges","archive_url":"https://api.github.com/repos/JuliaLang/julia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JuliaLang/julia/downloads","issues_url":"https://api.github.com/repos/JuliaLang/julia/issues{/number}","pulls_url":"https://api.github.com/repos/JuliaLang/julia/pulls{/number}","milestones_url":"https://api.github.com/repos/JuliaLang/julia/milestones{/number}","notifications_url":"https://api.github.com/repos/JuliaLang/julia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JuliaLang/julia/labels{/name}","releases_url":"https://api.github.com/repos/JuliaLang/julia/releases{/id}","deployments_url":"https://api.github.com/repos/JuliaLang/julia/deployments","created_at":"2011-04-21T07:01:50Z","updated_at":"2016-08-31T11:05:06Z","pushed_at":"2016-08-31T12:41:14Z","git_url":"git://github.com/JuliaLang/julia.git","ssh_url":"git@github.com:JuliaLang/julia.git","clone_url":"https://github.com/JuliaLang/julia.git","svn_url":"https://github.com/JuliaLang/julia","homepage":"http://julialang.org/","size":411607,"stargazers_count":7441,"watchers_count":7441,"language":"Julia","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1737,"mirror_url":null,"open_issues_count":1941,"forks":1737,"open_issues":1941,"watchers":7441,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/18289"},"html":{"href":"https://github.com/JuliaLang/julia/pull/18289"},"issue":{"href":"https://api.github.com/repos/JuliaLang/julia/issues/18289"},"comments":{"href":"https://api.github.com/repos/JuliaLang/julia/issues/18289/comments"},"review_comments":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/18289/comments"},"review_comment":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/JuliaLang/julia/pulls/18289/commits"},"statuses":{"href":"https://api.github.com/repos/JuliaLang/julia/statuses/77e8789c2d528c30c504564d670f2328fd72e884"}}}}
{"actor":{"display_login":"tkelman"}}