{"action":"created","comment":{"url":"https://api.github.com/repos/docker/docker/pulls/comments/76513622","id":76513622,"diff_hunk":"@@ -113,12 +121,228 @@ func (daemon *Daemon) GetCachedImage(imgID image.ID, config *containertypes.Conf\n \treturn getMatch(siblings)\n }\n \n+type daemonImageCacheForBuild struct {\n+\t// cacheFromImages here is a map of (provided) names to the actual images it represents\n+\tcacheFromImages map[string]*image.Image\n+\t// cacheFromImageHistories also provides a map back to a historyWithSourceT struct\n+\tcacheFromImageHistories map[string][]historyWithSourceT\n+\t// daemon stores a reference to the daemon that backs this cache\n+\tdaemon *Daemon\n+}\n+\n+func (daemon *Daemon) MakeImageCacheForBuild(cacheFrom []string) builder.ImageCacheForBuild {\n+\tcache := &daemonImageCacheForBuild{\n+\t\tdaemon:                  daemon,\n+\t\tcacheFromImages:         make(map[string]*image.Image),\n+\t\tcacheFromImageHistories: make(map[string][]historyWithSourceT),\n+\t}\n+\n+\t// for each cacheFrom image, set up the channels & coroutine for scrolling forward through\n+\t// its history and comparing it to what's being built\n+\tfor _, cacheFromImageName := range cacheFrom {\n+\t\tcacheFromImage, err := daemon.GetImage(cacheFromImageName)\n+\t\tif err != nil {\n+\t\t\tlogrus.Warnf(\"Could not look up %s for cache resolution, skipping: %s\", cacheFromImageName, err)\n+\t\t\tcontinue\n+\t\t}\n+\n+\t\tcache.cacheFromImages[cacheFromImageName] = cacheFromImage\n+\t\tcache.cacheFromImageHistories[cacheFromImageName] = makeHistoryWithSource(cacheFromImage)\n+\t}\n+\n+\treturn cache\n+}\n+\n+// In the history array, we have pairs of (command, resultingLayerID). What we actually want to be able\n+// to compare is pairs of (sourceLayerID, command), and if we have a match, consult resultingLayerID.\n+// We also don't directly have source/resultingLayerID, but rather a boolean \"did create new layer\" flag.\n+// Define a struct to store this mapping for convenience.\n+type historyWithSourceT struct {\n+\t// sourceLayerID is the layer on which the command was run. Empty digest if this is the first command or\n+\t// if nothing has actually been added to the rootfs yet.\n+\tsourceLayerID layer.DiffID\n+\t// cmd is the command which got run on sourceLayerID\n+\tcmd string\n+\t// resulingLayerID is the result of running cmd on sourceLayerID (might be the same as sourceLayerID)\n+\tresultingLayerID layer.DiffID\n+\t// createdAt is the time the history entry was created\n+\tcreatedAt time.Time\n+\t// history is the actual, underlying history entry\n+\thistory image.History\n+}\n+\n+func makeHistoryWithSource(image *image.Image) []historyWithSourceT {\n+\t// Let's make those structs now\n+\thistoryWithSource := make([]historyWithSourceT, len(image.History))\n+\tlayerIndex := -1\n+\tfor i, h := range image.History {\n+\n+\t\t// previous is layerIndex from previous iteration\n+\t\tif layerIndex == -1 {\n+\t\t\thistoryWithSource[i].sourceLayerID = digest.DigestSha256EmptyTar\n+\t\t} else {\n+\t\t\thistoryWithSource[i].sourceLayerID = image.RootFS.DiffIDs[layerIndex]\n+\t\t}\n+\n+\t\t// now increment, if needed, and look at the result layer ID\n+\t\tif !h.EmptyLayer {\n+\t\t\tlayerIndex = layerIndex + 1\n+\t\t}\n+\t\tif layerIndex == -1 {\n+\t\t\thistoryWithSource[i].resultingLayerID = digest.DigestSha256EmptyTar\n+\t\t} else {\n+\t\t\thistoryWithSource[i].resultingLayerID = image.RootFS.DiffIDs[layerIndex]\n+\t\t}\n+\n+\t\t// Copy the other history entries over I'm interested in\n+\t\thistoryWithSource[i].cmd = h.CreatedBy\n+\t\thistoryWithSource[i].createdAt = h.Created\n+\t\thistoryWithSource[i].history = h\n+\t}\n+\n+\treturn historyWithSource\n+}\n+\n // GetCachedImageOnBuild returns a reference to a cached image whose parent equals `parent`\n // and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error.\n-func (daemon *Daemon) GetCachedImageOnBuild(imgID string, cfg *containertypes.Config) (string, error) {\n-\tcache, err := daemon.GetCachedImage(image.ID(imgID), cfg)\n-\tif cache == nil || err != nil {\n+func (cache *daemonImageCacheForBuild) GetCachedImageOnBuild(imgID string, cfg *containertypes.Config) (string, error) {\n+\tcachedImage, err := cache.daemon.GetCachedImage(image.ID(imgID), cfg)\n+\tif err != nil {\n \t\treturn \"\", err\n \t}\n-\treturn cache.ID().String(), nil\n+\tif cachedImage != nil {\n+\t\t// We found a cache hit using the old parent image method\n+\t\treturn cachedImage.ID().String(), nil\n+\t}\n+\n+\t// We didn't find a cache hit using that method. Explore cacheFrom images for matching history\n+\tvar parentImage *image.Image\n+\tvar parentImageHistory []historyWithSourceT\n+\t// i.e. not FROM SCRATCH\n+\tif imgID != \"\" {\n+\t\tparentImage, err = cache.daemon.imageStore.Get(image.ID(imgID))\n+\t\tif err != nil {\n+\t\t\treturn \"\", err\n+\t\t}\n+\t\tparentImageHistory = makeHistoryWithSource(parentImage)\n+\t}\n+\n+\t// For each thing we are caching from, see if it matches parentImageHistory\n+\ttype matchStruct struct {","path":"daemon/image.go","position":135,"original_position":135,"commit_id":"a34650e137d63326e030c85a770c141107483a57","original_commit_id":"a34650e137d63326e030c85a770c141107483a57","user":{"login":"KJTsanaktsidis","id":1418177,"avatar_url":"https://avatars.githubusercontent.com/u/1418177?v=3","gravatar_id":"","url":"https://api.github.com/users/KJTsanaktsidis","html_url":"https://github.com/KJTsanaktsidis","followers_url":"https://api.github.com/users/KJTsanaktsidis/followers","following_url":"https://api.github.com/users/KJTsanaktsidis/following{/other_user}","gists_url":"https://api.github.com/users/KJTsanaktsidis/gists{/gist_id}","starred_url":"https://api.github.com/users/KJTsanaktsidis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/KJTsanaktsidis/subscriptions","organizations_url":"https://api.github.com/users/KJTsanaktsidis/orgs","repos_url":"https://api.github.com/users/KJTsanaktsidis/repos","events_url":"https://api.github.com/users/KJTsanaktsidis/events{/privacy}","received_events_url":"https://api.github.com/users/KJTsanaktsidis/received_events","type":"User","site_admin":false},"body":"Likewise, I can probably find a way to get rid of this type and rephrase the code in terms of existing types.","created_at":"2016-08-27T09:41:36Z","updated_at":"2016-08-27T09:41:36Z","html_url":"https://github.com/docker/docker/pull/24711#discussion_r76513622","pull_request_url":"https://api.github.com/repos/docker/docker/pulls/24711","_links":{"self":{"href":"https://api.github.com/repos/docker/docker/pulls/comments/76513622"},"html":{"href":"https://github.com/docker/docker/pull/24711#discussion_r76513622"},"pull_request":{"href":"https://api.github.com/repos/docker/docker/pulls/24711"}}},"pull_request":{"url":"https://api.github.com/repos/docker/docker/pulls/24711","id":77698474,"html_url":"https://github.com/docker/docker/pull/24711","diff_url":"https://github.com/docker/docker/pull/24711.diff","patch_url":"https://github.com/docker/docker/pull/24711.patch","issue_url":"https://api.github.com/repos/docker/docker/issues/24711","number":24711,"state":"open","locked":false,"title":"Proposal: using registry images for build caching","user":{"login":"KJTsanaktsidis","id":1418177,"avatar_url":"https://avatars.githubusercontent.com/u/1418177?v=3","gravatar_id":"","url":"https://api.github.com/users/KJTsanaktsidis","html_url":"https://github.com/KJTsanaktsidis","followers_url":"https://api.github.com/users/KJTsanaktsidis/followers","following_url":"https://api.github.com/users/KJTsanaktsidis/following{/other_user}","gists_url":"https://api.github.com/users/KJTsanaktsidis/gists{/gist_id}","starred_url":"https://api.github.com/users/KJTsanaktsidis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/KJTsanaktsidis/subscriptions","organizations_url":"https://api.github.com/users/KJTsanaktsidis/orgs","repos_url":"https://api.github.com/users/KJTsanaktsidis/repos","events_url":"https://api.github.com/users/KJTsanaktsidis/events{/privacy}","received_events_url":"https://api.github.com/users/KJTsanaktsidis/received_events","type":"User","site_admin":false},"body":"**- What I did**\r\nIn this PR, I propose a plugin mechanism for the daemon to request\r\ncached images from a plugin. I believe a plugin is appropriate here\r\nbecause there are several different ways such sharing might work\r\ndepending on the lifecycle of the machines being used.\r\n\r\n**- How I did it**\r\nN/A\r\n\r\n**- How to verify it**\r\nN/A\r\n\r\n**- Description for the changelog**\r\nProposal: plugin mechanism for sharing image cache between machines\r\n\r\n**- A picture of a cute animal (not mandatory but encouraged)**\r\n![sant01-10a-penguinonseal](https://cloud.githubusercontent.com/assets/1418177/16892352/26521348-4b53-11e6-9c24-ad5380b63ac8.jpg)\r\n\r\n\r\nAs has been discussed extensively in #20316 , Docker now no longer\r\nrestores parent image chains when pulling from a registry, and people\r\nwho relied on this to cache builds across multiple machines have had\r\ntheir build times blow out.\r\n\r\nI'm PR'ing this to get some feedback from the maintainers & community on\r\nthis idea. If people agree that this is the way forward, I'd like to\r\nstart working on this. It doesn't appear to me that the implementation\r\nof the plugin surface would be especially difficult or disruptive - but\r\nI want your thoughts on this :)\r\n\r\nEDIT: correct the issue link","created_at":"2016-07-16T02:45:22Z","updated_at":"2016-08-27T09:41:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4a0858cd40a80aef58257374fd98c3b83ef310dc","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/docker/docker/pulls/24711/commits","review_comments_url":"https://api.github.com/repos/docker/docker/pulls/24711/comments","review_comment_url":"https://api.github.com/repos/docker/docker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/docker/docker/issues/24711/comments","statuses_url":"https://api.github.com/repos/docker/docker/statuses/a34650e137d63326e030c85a770c141107483a57","head":{"label":"KJTsanaktsidis:image_cache_plugin_proposal","ref":"image_cache_plugin_proposal","sha":"a34650e137d63326e030c85a770c141107483a57","user":{"login":"KJTsanaktsidis","id":1418177,"avatar_url":"https://avatars.githubusercontent.com/u/1418177?v=3","gravatar_id":"","url":"https://api.github.com/users/KJTsanaktsidis","html_url":"https://github.com/KJTsanaktsidis","followers_url":"https://api.github.com/users/KJTsanaktsidis/followers","following_url":"https://api.github.com/users/KJTsanaktsidis/following{/other_user}","gists_url":"https://api.github.com/users/KJTsanaktsidis/gists{/gist_id}","starred_url":"https://api.github.com/users/KJTsanaktsidis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/KJTsanaktsidis/subscriptions","organizations_url":"https://api.github.com/users/KJTsanaktsidis/orgs","repos_url":"https://api.github.com/users/KJTsanaktsidis/repos","events_url":"https://api.github.com/users/KJTsanaktsidis/events{/privacy}","received_events_url":"https://api.github.com/users/KJTsanaktsidis/received_events","type":"User","site_admin":false},"repo":{"id":60445327,"name":"docker","full_name":"KJTsanaktsidis/docker","owner":{"login":"KJTsanaktsidis","id":1418177,"avatar_url":"https://avatars.githubusercontent.com/u/1418177?v=3","gravatar_id":"","url":"https://api.github.com/users/KJTsanaktsidis","html_url":"https://github.com/KJTsanaktsidis","followers_url":"https://api.github.com/users/KJTsanaktsidis/followers","following_url":"https://api.github.com/users/KJTsanaktsidis/following{/other_user}","gists_url":"https://api.github.com/users/KJTsanaktsidis/gists{/gist_id}","starred_url":"https://api.github.com/users/KJTsanaktsidis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/KJTsanaktsidis/subscriptions","organizations_url":"https://api.github.com/users/KJTsanaktsidis/orgs","repos_url":"https://api.github.com/users/KJTsanaktsidis/repos","events_url":"https://api.github.com/users/KJTsanaktsidis/events{/privacy}","received_events_url":"https://api.github.com/users/KJTsanaktsidis/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/KJTsanaktsidis/docker","description":"Docker - the open-source application container engine","fork":true,"url":"https://api.github.com/repos/KJTsanaktsidis/docker","forks_url":"https://api.github.com/repos/KJTsanaktsidis/docker/forks","keys_url":"https://api.github.com/repos/KJTsanaktsidis/docker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/KJTsanaktsidis/docker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/KJTsanaktsidis/docker/teams","hooks_url":"https://api.github.com/repos/KJTsanaktsidis/docker/hooks","issue_events_url":"https://api.github.com/repos/KJTsanaktsidis/docker/issues/events{/number}","events_url":"https://api.github.com/repos/KJTsanaktsidis/docker/events","assignees_url":"https://api.github.com/repos/KJTsanaktsidis/docker/assignees{/user}","branches_url":"https://api.github.com/repos/KJTsanaktsidis/docker/branches{/branch}","tags_url":"https://api.github.com/repos/KJTsanaktsidis/docker/tags","blobs_url":"https://api.github.com/repos/KJTsanaktsidis/docker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/KJTsanaktsidis/docker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/KJTsanaktsidis/docker/git/refs{/sha}","trees_url":"https://api.github.com/repos/KJTsanaktsidis/docker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/KJTsanaktsidis/docker/statuses/{sha}","languages_url":"https://api.github.com/repos/KJTsanaktsidis/docker/languages","stargazers_url":"https://api.github.com/repos/KJTsanaktsidis/docker/stargazers","contributors_url":"https://api.github.com/repos/KJTsanaktsidis/docker/contributors","subscribers_url":"https://api.github.com/repos/KJTsanaktsidis/docker/subscribers","subscription_url":"https://api.github.com/repos/KJTsanaktsidis/docker/subscription","commits_url":"https://api.github.com/repos/KJTsanaktsidis/docker/commits{/sha}","git_commits_url":"https://api.github.com/repos/KJTsanaktsidis/docker/git/commits{/sha}","comments_url":"https://api.github.com/repos/KJTsanaktsidis/docker/comments{/number}","issue_comment_url":"https://api.github.com/repos/KJTsanaktsidis/docker/issues/comments{/number}","contents_url":"https://api.github.com/repos/KJTsanaktsidis/docker/contents/{+path}","compare_url":"https://api.github.com/repos/KJTsanaktsidis/docker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/KJTsanaktsidis/docker/merges","archive_url":"https://api.github.com/repos/KJTsanaktsidis/docker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/KJTsanaktsidis/docker/downloads","issues_url":"https://api.github.com/repos/KJTsanaktsidis/docker/issues{/number}","pulls_url":"https://api.github.com/repos/KJTsanaktsidis/docker/pulls{/number}","milestones_url":"https://api.github.com/repos/KJTsanaktsidis/docker/milestones{/number}","notifications_url":"https://api.github.com/repos/KJTsanaktsidis/docker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/KJTsanaktsidis/docker/labels{/name}","releases_url":"https://api.github.com/repos/KJTsanaktsidis/docker/releases{/id}","deployments_url":"https://api.github.com/repos/KJTsanaktsidis/docker/deployments","created_at":"2016-06-05T05:49:08Z","updated_at":"2016-06-05T05:49:21Z","pushed_at":"2016-08-27T09:36:25Z","git_url":"git://github.com/KJTsanaktsidis/docker.git","ssh_url":"git@github.com:KJTsanaktsidis/docker.git","clone_url":"https://github.com/KJTsanaktsidis/docker.git","svn_url":"https://github.com/KJTsanaktsidis/docker","homepage":"http://www.docker.com","size":95533,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"docker:master","ref":"master","sha":"6072e6bfd20598f4e9cb28d32c09bfddec6df76c","user":{"login":"docker","id":5429470,"avatar_url":"https://avatars.githubusercontent.com/u/5429470?v=3","gravatar_id":"","url":"https://api.github.com/users/docker","html_url":"https://github.com/docker","followers_url":"https://api.github.com/users/docker/followers","following_url":"https://api.github.com/users/docker/following{/other_user}","gists_url":"https://api.github.com/users/docker/gists{/gist_id}","starred_url":"https://api.github.com/users/docker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/docker/subscriptions","organizations_url":"https://api.github.com/users/docker/orgs","repos_url":"https://api.github.com/users/docker/repos","events_url":"https://api.github.com/users/docker/events{/privacy}","received_events_url":"https://api.github.com/users/docker/received_events","type":"Organization","site_admin":false},"repo":{"id":7691631,"name":"docker","full_name":"docker/docker","owner":{"login":"docker","id":5429470,"avatar_url":"https://avatars.githubusercontent.com/u/5429470?v=3","gravatar_id":"","url":"https://api.github.com/users/docker","html_url":"https://github.com/docker","followers_url":"https://api.github.com/users/docker/followers","following_url":"https://api.github.com/users/docker/following{/other_user}","gists_url":"https://api.github.com/users/docker/gists{/gist_id}","starred_url":"https://api.github.com/users/docker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/docker/subscriptions","organizations_url":"https://api.github.com/users/docker/orgs","repos_url":"https://api.github.com/users/docker/repos","events_url":"https://api.github.com/users/docker/events{/privacy}","received_events_url":"https://api.github.com/users/docker/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/docker/docker","description":"Docker - the open-source application container engine","fork":false,"url":"https://api.github.com/repos/docker/docker","forks_url":"https://api.github.com/repos/docker/docker/forks","keys_url":"https://api.github.com/repos/docker/docker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/docker/docker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/docker/docker/teams","hooks_url":"https://api.github.com/repos/docker/docker/hooks","issue_events_url":"https://api.github.com/repos/docker/docker/issues/events{/number}","events_url":"https://api.github.com/repos/docker/docker/events","assignees_url":"https://api.github.com/repos/docker/docker/assignees{/user}","branches_url":"https://api.github.com/repos/docker/docker/branches{/branch}","tags_url":"https://api.github.com/repos/docker/docker/tags","blobs_url":"https://api.github.com/repos/docker/docker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/docker/docker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/docker/docker/git/refs{/sha}","trees_url":"https://api.github.com/repos/docker/docker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/docker/docker/statuses/{sha}","languages_url":"https://api.github.com/repos/docker/docker/languages","stargazers_url":"https://api.github.com/repos/docker/docker/stargazers","contributors_url":"https://api.github.com/repos/docker/docker/contributors","subscribers_url":"https://api.github.com/repos/docker/docker/subscribers","subscription_url":"https://api.github.com/repos/docker/docker/subscription","commits_url":"https://api.github.com/repos/docker/docker/commits{/sha}","git_commits_url":"https://api.github.com/repos/docker/docker/git/commits{/sha}","comments_url":"https://api.github.com/repos/docker/docker/comments{/number}","issue_comment_url":"https://api.github.com/repos/docker/docker/issues/comments{/number}","contents_url":"https://api.github.com/repos/docker/docker/contents/{+path}","compare_url":"https://api.github.com/repos/docker/docker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/docker/docker/merges","archive_url":"https://api.github.com/repos/docker/docker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/docker/docker/downloads","issues_url":"https://api.github.com/repos/docker/docker/issues{/number}","pulls_url":"https://api.github.com/repos/docker/docker/pulls{/number}","milestones_url":"https://api.github.com/repos/docker/docker/milestones{/number}","notifications_url":"https://api.github.com/repos/docker/docker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/docker/docker/labels{/name}","releases_url":"https://api.github.com/repos/docker/docker/releases{/id}","deployments_url":"https://api.github.com/repos/docker/docker/deployments","created_at":"2013-01-18T18:10:57Z","updated_at":"2016-08-27T08:34:05Z","pushed_at":"2016-08-27T09:36:27Z","git_url":"git://github.com/docker/docker.git","ssh_url":"git@github.com:docker/docker.git","clone_url":"https://github.com/docker/docker.git","svn_url":"https://github.com/docker/docker","homepage":"http://www.docker.com","size":206418,"stargazers_count":34379,"watchers_count":34379,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":10070,"mirror_url":null,"open_issues_count":2168,"forks":10070,"open_issues":2168,"watchers":34379,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/docker/docker/pulls/24711"},"html":{"href":"https://github.com/docker/docker/pull/24711"},"issue":{"href":"https://api.github.com/repos/docker/docker/issues/24711"},"comments":{"href":"https://api.github.com/repos/docker/docker/issues/24711/comments"},"review_comments":{"href":"https://api.github.com/repos/docker/docker/pulls/24711/comments"},"review_comment":{"href":"https://api.github.com/repos/docker/docker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/docker/docker/pulls/24711/commits"},"statuses":{"href":"https://api.github.com/repos/docker/docker/statuses/a34650e137d63326e030c85a770c141107483a57"}}}}
{"actor":{"display_login":"KJTsanaktsidis"}}