{"action":"created","comment":{"url":"https://api.github.com/repos/containers/image/pulls/comments/77038032","id":77038032,"diff_hunk":"@@ -0,0 +1,411 @@\n+package storage\n+\n+import (\n+\t\"bytes\"\n+\t\"crypto/sha256\"\n+\t\"encoding/hex\"\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"io/ioutil\"\n+\t\"time\"\n+\n+\t\"github.com/Sirupsen/logrus\"\n+\t\"github.com/containers/image/image\"\n+\t\"github.com/containers/image/types\"\n+\t\"github.com/containers/storage/pkg/archive\"\n+\t\"github.com/containers/storage/storage\"\n+)\n+\n+type storageImage struct {\n+\ttransport      *storageTransport\n+\timageRef       *storageReference\n+\tTag            string            `json:\"tag,omitifempty\"`\n+\tCreated        time.Time         `json:\"created-time,omitifempty\"`\n+\tID             string            `json:\"id\"`\n+\tBlobList       []string          `json:\"blob-list,omitifempty\"`    // Ordered list of every blob the image has been told to handle\n+\tLayers         map[string]string `json:\"layers,omitifempty\"`       // Map from names of blobs that are layers to layer IDs\n+\tBlobData       map[string][]byte `json:\"-\"`                        // Map from names of blobs that aren't layers to contents, temporary\n+\tBlobDigestData map[string]string `json:\"blob-digests,omitifempty\"` // Map from names of blobs to digests\n+\tSignatureSizes []int             `json:\"signature-sizes\"`          // List of sizes of each signature slice\n+}\n+\n+type storageLayerMetadata struct {\n+\tCompression archive.Compression `json:\"compression\"`\n+}\n+\n+// newImageSource sets us up to read out an image, which we assume exists.\n+func newImageSource(transport *storageTransport, imageRef *storageReference) *storageImage {\n+\tref := imageRef.reference\n+\ttag := \"\"\n+\tif imageRef.id != \"\" {\n+\t\tref = imageRef.id\n+\t\ttag = imageRef.reference\n+\t}\n+\timg, err := transport.mall.GetImage(ref)\n+\tif err != nil {\n+\t\treturn nil\n+\t}\n+\timage := storageImage{\n+\t\ttransport:      transport,\n+\t\timageRef:       imageRef,\n+\t\tTag:            tag,\n+\t\tCreated:        time.Now(),\n+\t\tID:             img.ID,\n+\t\tBlobList:       []string{},\n+\t\tLayers:         make(map[string]string),\n+\t\tBlobData:       make(map[string][]byte),\n+\t\tBlobDigestData: make(map[string]string),\n+\t}\n+\timage.loadMetadata()\n+\treturn &image\n+}\n+\n+// newImageDestination sets us up to write a new image.\n+func newImageDestination(transport *storageTransport, imageRef *storageReference) *storageImage {\n+\t// We set the ID if the reference we got looked like one, since we take\n+\t// that as an indication that it's going to end up with the same\n+\t// contents as an image we already have.  If the reference looks more\n+\t// like a name, we don't know yet if it'll be exactly the same as, or\n+\t// an updated version of, an image we might already have, so we have to\n+\t// err on the side of caution and create a new image which will be\n+\t// assigned the name as its tag.\n+\timage := storageImage{\n+\t\ttransport:      transport,\n+\t\timageRef:       imageRef,\n+\t\tTag:            imageRef.reference,\n+\t\tCreated:        time.Now(),\n+\t\tID:             imageRef.id,\n+\t\tBlobList:       []string{},\n+\t\tLayers:         make(map[string]string),\n+\t\tBlobData:       make(map[string][]byte),\n+\t\tBlobDigestData: make(map[string]string),\n+\t}\n+\treturn &image\n+}\n+\n+func newImage(transport *storageTransport, imageRef *storageReference) *storageImage {\n+\treturn newImageSource(transport, imageRef)\n+}\n+\n+func (s *storageImage) loadMetadata() {\n+\tif s.ID != \"\" {\n+\t\timage, err := s.transport.mall.GetImage(s.ID)\n+\t\tif image != nil && err == nil {\n+\t\t\tjson.Unmarshal([]byte(image.Metadata), s)\n+\t\t}\n+\t}\n+}\n+\n+func (s *storageImage) saveMetadata() {\n+\tif s.ID != \"\" {\n+\t\tif metadata, err := json.Marshal(s); len(metadata) != 0 && err == nil {\n+\t\t\tistore, err := s.transport.mall.GetImageStore()\n+\t\t\tif istore != nil && err == nil {\n+\t\t\t\tistore.SetMetadata(s.ID, string(metadata))\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+func (s *storageImage) Reference() types.ImageReference {\n+\ts.ID = s.imageRef.ID()\n+\treturn s.imageRef\n+}\n+\n+// PutBlob is used to both store filesystem layers and binary data that is part\n+// of the image.\n+func (s *storageImage) PutBlob(digest string, stream io.Reader) error {\n+\t// Try to read an initial snippet of the blob.\n+\theader := make([]byte, 10240)\n+\tn, err := stream.Read(header)\n+\tif err != nil && err != io.EOF {\n+\t\treturn err\n+\t}\n+\t// Set up to read the whole blob (the initial snippet, plus the rest)\n+\t// while digesting it with sha256.\n+\thasher := sha256.New()\n+\tdefragmented := io.MultiReader(bytes.NewBuffer(header[:n]), stream)\n+\tmulti := io.TeeReader(defragmented, hasher)\n+\tif (n > 0) && archive.IsArchive(header[:n]) {\n+\t\t// It's a filesystem layer.  If it's not the first one in the\n+\t\t// image, we assume that the most recently added layer is its\n+\t\t// parent.","path":"storage/storage_image.go","position":133,"original_position":133,"commit_id":"3cbb921b6f0b2dc2d8f170db15f2781ad8ef0744","original_commit_id":"3cbb921b6f0b2dc2d8f170db15f2781ad8ef0744","user":{"login":"mtrmac","id":7052838,"avatar_url":"https://avatars.githubusercontent.com/u/7052838?v=3","gravatar_id":"","url":"https://api.github.com/users/mtrmac","html_url":"https://github.com/mtrmac","followers_url":"https://api.github.com/users/mtrmac/followers","following_url":"https://api.github.com/users/mtrmac/following{/other_user}","gists_url":"https://api.github.com/users/mtrmac/gists{/gist_id}","starred_url":"https://api.github.com/users/mtrmac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mtrmac/subscriptions","organizations_url":"https://api.github.com/users/mtrmac/orgs","repos_url":"https://api.github.com/users/mtrmac/repos","events_url":"https://api.github.com/users/mtrmac/events{/privacy}","received_events_url":"https://api.github.com/users/mtrmac/received_events","type":"User","site_admin":false},"body":"Sadly, looking at the current `genericImage.BlobDigests`, the order is not currently preserved on input. That can change of course, and I think `docker-daemon:` will need this to change as well.\r\n\r\n---\r\n\r\nFor these kinds of API changes, it would be very convenient to have both the definition of the interfaces, the transports implementing the interfaces, and the core of `skopeo copy`, the primary user, in a single repo — then we could change the `ImageSource`/`ImageDestination` interfaces without breaking anything else and without needing to keep a `skopeo` branch+PR in sync.\r\n\r\nhttps://github.com/containers/image/pull/56 + https://github.com/projectatomic/skopeo/pull/175 does that, so if this PR ended up changing the types, perhaps it would be practical to base it on that?\r\n\r\n(Warning: #56 is being rebased as other PRs are merged; but I don't expect it to substantially change.)\r\n\r\n@runcom Note that #56 currently depends on #52 , primarily for the `types.SystemContext` API changes; I could split the API changes from #52 so that #52 is only the lookaside implementation, which would allow merging the API changes + `copy.go` before reviewing the lookaside—if that helped you in any way.","created_at":"2016-08-31T17:53:29Z","updated_at":"2016-08-31T17:53:29Z","html_url":"https://github.com/containers/image/pull/63#discussion_r77038032","pull_request_url":"https://api.github.com/repos/containers/image/pulls/63","_links":{"self":{"href":"https://api.github.com/repos/containers/image/pulls/comments/77038032"},"html":{"href":"https://github.com/containers/image/pull/63#discussion_r77038032"},"pull_request":{"href":"https://api.github.com/repos/containers/image/pulls/63"}}},"pull_request":{"url":"https://api.github.com/repos/containers/image/pulls/63","id":83463408,"html_url":"https://github.com/containers/image/pull/63","diff_url":"https://github.com/containers/image/pull/63.diff","patch_url":"https://github.com/containers/image/pull/63.patch","issue_url":"https://api.github.com/repos/containers/image/issues/63","number":63,"state":"open","locked":false,"title":"Add support for containers/storage","user":{"login":"nalind","id":1200159,"avatar_url":"https://avatars.githubusercontent.com/u/1200159?v=3","gravatar_id":"","url":"https://api.github.com/users/nalind","html_url":"https://github.com/nalind","followers_url":"https://api.github.com/users/nalind/followers","following_url":"https://api.github.com/users/nalind/following{/other_user}","gists_url":"https://api.github.com/users/nalind/gists{/gist_id}","starred_url":"https://api.github.com/users/nalind/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nalind/subscriptions","organizations_url":"https://api.github.com/users/nalind/orgs","repos_url":"https://api.github.com/users/nalind/repos","events_url":"https://api.github.com/users/nalind/events{/privacy}","received_events_url":"https://api.github.com/users/nalind/received_events","type":"User","site_admin":false},"body":"This patch adds containers/storage as a backend type called 'oci-storage', using a storage.Mall initialized with the library's default settings, or using an externally-initialized storage.Mall.\r\n\r\nAn image's blobs are stored either as oci-storage layers (if they look like filesystem layers) or (alongside the manifest and signatures) as named big data items associated with the image.\r\n\r\nRecord-keeping (so that we can remember which blobs were layers and which weren't) are encoded as a JSON object which is stored as the oci-storage image's metadata.\r\n\r\nThe type of compression which we detected around a layer blob when it was imported is encoded as a field in a JSON object which is stored as the oci-storage layer object's metadata, and is reapplied if we need to export the layer later.\r\n\r\nIf we find ourselves importing an image with the same ID as an image that we already have, we return an error.  If we find ourselves importing an image which contains layers which we already have, we flag\r\nan error if its contents differ from the one we already have, otherwise we accept it.\r\n\r\nIf we find ourselves importing an image that wants to be tagged with a name that is already in use, the name is assigned to the new image and the old image remains otherwise unmodified.","created_at":"2016-08-31T14:27:56Z","updated_at":"2016-08-31T17:53:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"51a64c9c1bfc3b8eb3bcd653af5b31c300dd18fd","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/containers/image/pulls/63/commits","review_comments_url":"https://api.github.com/repos/containers/image/pulls/63/comments","review_comment_url":"https://api.github.com/repos/containers/image/pulls/comments{/number}","comments_url":"https://api.github.com/repos/containers/image/issues/63/comments","statuses_url":"https://api.github.com/repos/containers/image/statuses/3cbb921b6f0b2dc2d8f170db15f2781ad8ef0744","head":{"label":"nalind:add-storage-transport","ref":"add-storage-transport","sha":"3cbb921b6f0b2dc2d8f170db15f2781ad8ef0744","user":{"login":"nalind","id":1200159,"avatar_url":"https://avatars.githubusercontent.com/u/1200159?v=3","gravatar_id":"","url":"https://api.github.com/users/nalind","html_url":"https://github.com/nalind","followers_url":"https://api.github.com/users/nalind/followers","following_url":"https://api.github.com/users/nalind/following{/other_user}","gists_url":"https://api.github.com/users/nalind/gists{/gist_id}","starred_url":"https://api.github.com/users/nalind/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nalind/subscriptions","organizations_url":"https://api.github.com/users/nalind/orgs","repos_url":"https://api.github.com/users/nalind/repos","events_url":"https://api.github.com/users/nalind/events{/privacy}","received_events_url":"https://api.github.com/users/nalind/received_events","type":"User","site_admin":false},"repo":{"id":65415381,"name":"image","full_name":"nalind/image","owner":{"login":"nalind","id":1200159,"avatar_url":"https://avatars.githubusercontent.com/u/1200159?v=3","gravatar_id":"","url":"https://api.github.com/users/nalind","html_url":"https://github.com/nalind","followers_url":"https://api.github.com/users/nalind/followers","following_url":"https://api.github.com/users/nalind/following{/other_user}","gists_url":"https://api.github.com/users/nalind/gists{/gist_id}","starred_url":"https://api.github.com/users/nalind/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nalind/subscriptions","organizations_url":"https://api.github.com/users/nalind/orgs","repos_url":"https://api.github.com/users/nalind/repos","events_url":"https://api.github.com/users/nalind/events{/privacy}","received_events_url":"https://api.github.com/users/nalind/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nalind/image","description":"Work with containers' images","fork":true,"url":"https://api.github.com/repos/nalind/image","forks_url":"https://api.github.com/repos/nalind/image/forks","keys_url":"https://api.github.com/repos/nalind/image/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nalind/image/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nalind/image/teams","hooks_url":"https://api.github.com/repos/nalind/image/hooks","issue_events_url":"https://api.github.com/repos/nalind/image/issues/events{/number}","events_url":"https://api.github.com/repos/nalind/image/events","assignees_url":"https://api.github.com/repos/nalind/image/assignees{/user}","branches_url":"https://api.github.com/repos/nalind/image/branches{/branch}","tags_url":"https://api.github.com/repos/nalind/image/tags","blobs_url":"https://api.github.com/repos/nalind/image/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nalind/image/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nalind/image/git/refs{/sha}","trees_url":"https://api.github.com/repos/nalind/image/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nalind/image/statuses/{sha}","languages_url":"https://api.github.com/repos/nalind/image/languages","stargazers_url":"https://api.github.com/repos/nalind/image/stargazers","contributors_url":"https://api.github.com/repos/nalind/image/contributors","subscribers_url":"https://api.github.com/repos/nalind/image/subscribers","subscription_url":"https://api.github.com/repos/nalind/image/subscription","commits_url":"https://api.github.com/repos/nalind/image/commits{/sha}","git_commits_url":"https://api.github.com/repos/nalind/image/git/commits{/sha}","comments_url":"https://api.github.com/repos/nalind/image/comments{/number}","issue_comment_url":"https://api.github.com/repos/nalind/image/issues/comments{/number}","contents_url":"https://api.github.com/repos/nalind/image/contents/{+path}","compare_url":"https://api.github.com/repos/nalind/image/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nalind/image/merges","archive_url":"https://api.github.com/repos/nalind/image/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nalind/image/downloads","issues_url":"https://api.github.com/repos/nalind/image/issues{/number}","pulls_url":"https://api.github.com/repos/nalind/image/pulls{/number}","milestones_url":"https://api.github.com/repos/nalind/image/milestones{/number}","notifications_url":"https://api.github.com/repos/nalind/image/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nalind/image/labels{/name}","releases_url":"https://api.github.com/repos/nalind/image/releases{/id}","deployments_url":"https://api.github.com/repos/nalind/image/deployments","created_at":"2016-08-10T20:46:37Z","updated_at":"2016-08-10T20:46:38Z","pushed_at":"2016-08-31T15:32:48Z","git_url":"git://github.com/nalind/image.git","ssh_url":"git@github.com:nalind/image.git","clone_url":"https://github.com/nalind/image.git","svn_url":"https://github.com/nalind/image","homepage":"","size":1799,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"containers:master","ref":"master","sha":"5865cfca97861810b9b04fe7d486b065922c70af","user":{"login":"containers","id":5874934,"avatar_url":"https://avatars.githubusercontent.com/u/5874934?v=3","gravatar_id":"","url":"https://api.github.com/users/containers","html_url":"https://github.com/containers","followers_url":"https://api.github.com/users/containers/followers","following_url":"https://api.github.com/users/containers/following{/other_user}","gists_url":"https://api.github.com/users/containers/gists{/gist_id}","starred_url":"https://api.github.com/users/containers/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/containers/subscriptions","organizations_url":"https://api.github.com/users/containers/orgs","repos_url":"https://api.github.com/users/containers/repos","events_url":"https://api.github.com/users/containers/events{/privacy}","received_events_url":"https://api.github.com/users/containers/received_events","type":"Organization","site_admin":false},"repo":{"id":61739430,"name":"image","full_name":"containers/image","owner":{"login":"containers","id":5874934,"avatar_url":"https://avatars.githubusercontent.com/u/5874934?v=3","gravatar_id":"","url":"https://api.github.com/users/containers","html_url":"https://github.com/containers","followers_url":"https://api.github.com/users/containers/followers","following_url":"https://api.github.com/users/containers/following{/other_user}","gists_url":"https://api.github.com/users/containers/gists{/gist_id}","starred_url":"https://api.github.com/users/containers/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/containers/subscriptions","organizations_url":"https://api.github.com/users/containers/orgs","repos_url":"https://api.github.com/users/containers/repos","events_url":"https://api.github.com/users/containers/events{/privacy}","received_events_url":"https://api.github.com/users/containers/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/containers/image","description":"Work with containers' images","fork":false,"url":"https://api.github.com/repos/containers/image","forks_url":"https://api.github.com/repos/containers/image/forks","keys_url":"https://api.github.com/repos/containers/image/keys{/key_id}","collaborators_url":"https://api.github.com/repos/containers/image/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/containers/image/teams","hooks_url":"https://api.github.com/repos/containers/image/hooks","issue_events_url":"https://api.github.com/repos/containers/image/issues/events{/number}","events_url":"https://api.github.com/repos/containers/image/events","assignees_url":"https://api.github.com/repos/containers/image/assignees{/user}","branches_url":"https://api.github.com/repos/containers/image/branches{/branch}","tags_url":"https://api.github.com/repos/containers/image/tags","blobs_url":"https://api.github.com/repos/containers/image/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/containers/image/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/containers/image/git/refs{/sha}","trees_url":"https://api.github.com/repos/containers/image/git/trees{/sha}","statuses_url":"https://api.github.com/repos/containers/image/statuses/{sha}","languages_url":"https://api.github.com/repos/containers/image/languages","stargazers_url":"https://api.github.com/repos/containers/image/stargazers","contributors_url":"https://api.github.com/repos/containers/image/contributors","subscribers_url":"https://api.github.com/repos/containers/image/subscribers","subscription_url":"https://api.github.com/repos/containers/image/subscription","commits_url":"https://api.github.com/repos/containers/image/commits{/sha}","git_commits_url":"https://api.github.com/repos/containers/image/git/commits{/sha}","comments_url":"https://api.github.com/repos/containers/image/comments{/number}","issue_comment_url":"https://api.github.com/repos/containers/image/issues/comments{/number}","contents_url":"https://api.github.com/repos/containers/image/contents/{+path}","compare_url":"https://api.github.com/repos/containers/image/compare/{base}...{head}","merges_url":"https://api.github.com/repos/containers/image/merges","archive_url":"https://api.github.com/repos/containers/image/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/containers/image/downloads","issues_url":"https://api.github.com/repos/containers/image/issues{/number}","pulls_url":"https://api.github.com/repos/containers/image/pulls{/number}","milestones_url":"https://api.github.com/repos/containers/image/milestones{/number}","notifications_url":"https://api.github.com/repos/containers/image/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/containers/image/labels{/name}","releases_url":"https://api.github.com/repos/containers/image/releases{/id}","deployments_url":"https://api.github.com/repos/containers/image/deployments","created_at":"2016-06-22T17:46:41Z","updated_at":"2016-08-31T08:01:35Z","pushed_at":"2016-08-31T15:32:49Z","git_url":"git://github.com/containers/image.git","ssh_url":"git@github.com:containers/image.git","clone_url":"https://github.com/containers/image.git","svn_url":"https://github.com/containers/image","homepage":"","size":1853,"stargazers_count":22,"watchers_count":22,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":10,"mirror_url":null,"open_issues_count":18,"forks":10,"open_issues":18,"watchers":22,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/containers/image/pulls/63"},"html":{"href":"https://github.com/containers/image/pull/63"},"issue":{"href":"https://api.github.com/repos/containers/image/issues/63"},"comments":{"href":"https://api.github.com/repos/containers/image/issues/63/comments"},"review_comments":{"href":"https://api.github.com/repos/containers/image/pulls/63/comments"},"review_comment":{"href":"https://api.github.com/repos/containers/image/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/containers/image/pulls/63/commits"},"statuses":{"href":"https://api.github.com/repos/containers/image/statuses/3cbb921b6f0b2dc2d8f170db15f2781ad8ef0744"}}}}
{"actor":{"display_login":"mtrmac"}}