{"action":"created","comment":{"url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/comments/77394226","id":77394226,"diff_hunk":"@@ -0,0 +1,100 @@\n+(ns hitchhiker.s3\n+  (:require [amazonica.aws.s3 :as s3]\n+            [clojure.string :refer [split]]\n+            [hitchhiker.tree.core :as core]\n+            [taoensso.nippy :as nippy])\n+  (:import [com.google.common.io ByteStreams]\n+           [java.util UUID]\n+           [java.io ByteArrayInputStream]))\n+\n+;;; Main node data stored in uuid key\n+;;;   bucket-name/8E3806E4-865D-43C8-A823-9CCF7D9D88CB\n+;;; References from node -> node uses sub object:\n+;;;   bucket-name/F9FD37E5-5BF3-4681-93A0-A56E9823A068/->2793532B-8FC9-4E0D-8CD4-19AE87F3133E\n+;;; References to node uses sub object:\n+;;;   bucket-name/0834FB96-CC7A-4531-8A8E-B9FB811F5D5B/<-A38D2654-EDF2-460A-9D88-388DEC77AA05\n+\n+(defrecord S3Addr [last-key bucket key storage-addr]\n+  core/IResolve\n+  (dirty? [_] false)\n+  (last-key [_] last-key)\n+  (resolve [_]\n+    (with-open [in (:input-stream (s3/get-object bucket key))]\n+      (nippy/thaw (ByteStreams/toByteArray in)))))\n+\n+(defn synthesize-storage-addr\n+  [bucket key]\n+  (doto (promise)\n+    (deliver {:bucket bucket :key key})))\n+\n+(defn s3-addr\n+  [last-key bucket key]\n+  (->S3Addr last-key bucket key (synthesize-storage-addr bucket key)))\n+\n+(nippy/extend-thaw :b-tree/s3-addr\n+                   [data-input]\n+                   (let [last-key (nippy/thaw-from-in! data-input)\n+                         bucket (nippy/thaw-from-in! data-input)\n+                         key (nippy/thaw-from-in! data-input)]\n+                     (s3-addr last-key bucket key)))\n+\n+(nippy/extend-freeze S3Addr :b-tree/s3-addr\n+                     [{:keys [last-key bucket key]} data-output]\n+                     (nippy/freeze-to-out! data-output last-key)\n+                     (nippy/freeze-to-out! data-output bucket)\n+                     (nippy/freeze-to-out! data-output key))\n+\n+(defn write-object\n+  [bucket key bytes]\n+  (println \"write-object\" bucket key (count bytes))\n+  (s3/put-object :bucket-name bucket\n+                 :key key\n+                 :metadata {:content-length (count bytes)}\n+                 :input-stream (ByteArrayInputStream. bytes)))\n+\n+(defn delete-object\n+  [bucket key]\n+  (doall (for [other-key (map :key (:object-summaries\n+                                    (s3/list-objects :bucket-name bucket\n+                                                     :prefix (str key \"/->\"))))]\n+           (s3/delete-object :bucket-name bucket\n+                             :key (str (last (split other-key \"/->\")) \"/<-\" key))\n+           ; TODO: delete other-key if no refs?\n+           ))\n+  (doall (for [other-key (map :key (:object-summaries\n+                                    (s3/list-objects :bucket-name bucket\n+                                                     :prefix (str key \"/<-\"))))]\n+           (s3/delete-object :bucket-name bucket\n+                             :key (str (last (split other-key \"/<-\")) \"/->\" key))))\n+  (s3/delete-object :bucket-name bucket :key key))\n+\n+(defn add-refs\n+  [node-key child-keys]\n+  (doall\n+   (for [{:keys [bucket key]} child-keys]\n+     (do\n+       (write-object bucket (str node-key \"/->\" key) (byte-array 0))\n+       (write-object bucket (str key \"/<-\" node-key) (byte-array 0))))))\n+\n+(defrecord S3Backend [#_service bucket]\n+  core/IBackend\n+  (new-session [_]\n+    (atom {:writes 0 :deletes 0}))","path":"src/hitchhiker/s3.clj","position":82,"original_position":82,"commit_id":"261e764878971214b5f08d7c266a764ab8b22546","original_commit_id":"261e764878971214b5f08d7c266a764ab8b22546","user":{"login":"arichiardi","id":1724788,"avatar_url":"https://avatars.githubusercontent.com/u/1724788?v=3","gravatar_id":"","url":"https://api.github.com/users/arichiardi","html_url":"https://github.com/arichiardi","followers_url":"https://api.github.com/users/arichiardi/followers","following_url":"https://api.github.com/users/arichiardi/following{/other_user}","gists_url":"https://api.github.com/users/arichiardi/gists{/gist_id}","starred_url":"https://api.github.com/users/arichiardi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arichiardi/subscriptions","organizations_url":"https://api.github.com/users/arichiardi/orgs","repos_url":"https://api.github.com/users/arichiardi/repos","events_url":"https://api.github.com/users/arichiardi/events{/privacy}","received_events_url":"https://api.github.com/users/arichiardi/received_events","type":"User","site_admin":false},"body":"I think `confetti` does some diffing for you though, code here: https://github.com/confetti-clj/s3-deploy/blob/master/src/confetti/s3_deploy.clj. This is a great idea, keep it up!","created_at":"2016-09-02T18:43:04Z","updated_at":"2016-09-02T18:43:04Z","html_url":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23#discussion_r77394226","pull_request_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23","_links":{"self":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/comments/77394226"},"html":{"href":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23#discussion_r77394226"},"pull_request":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23"}}},"pull_request":{"url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23","id":83828973,"html_url":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23","diff_url":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23.diff","patch_url":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23.patch","issue_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/23","number":23,"state":"open","locked":false,"title":"An S3 backend.","user":{"login":"csm","id":39441,"avatar_url":"https://avatars.githubusercontent.com/u/39441?v=3","gravatar_id":"","url":"https://api.github.com/users/csm","html_url":"https://github.com/csm","followers_url":"https://api.github.com/users/csm/followers","following_url":"https://api.github.com/users/csm/following{/other_user}","gists_url":"https://api.github.com/users/csm/gists{/gist_id}","starred_url":"https://api.github.com/users/csm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/csm/subscriptions","organizations_url":"https://api.github.com/users/csm/orgs","repos_url":"https://api.github.com/users/csm/repos","events_url":"https://api.github.com/users/csm/events{/privacy}","received_events_url":"https://api.github.com/users/csm/received_events","type":"User","site_admin":false},"body":"This is all pretty experimental, and I'm not sure I understand everything a backend needs to do. It's totally naive, but this might be the start of something useful. Kind of cool that it only took ~100 lines of code to do 8-)","created_at":"2016-09-02T16:24:29Z","updated_at":"2016-09-02T18:43:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"02ab9282df552f8ae53d6c1072327c3fc78452d9","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23/commits","review_comments_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23/comments","review_comment_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/comments{/number}","comments_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/23/comments","statuses_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/statuses/261e764878971214b5f08d7c266a764ab8b22546","head":{"label":"csm:s3","ref":"s3","sha":"261e764878971214b5f08d7c266a764ab8b22546","user":{"login":"csm","id":39441,"avatar_url":"https://avatars.githubusercontent.com/u/39441?v=3","gravatar_id":"","url":"https://api.github.com/users/csm","html_url":"https://github.com/csm","followers_url":"https://api.github.com/users/csm/followers","following_url":"https://api.github.com/users/csm/following{/other_user}","gists_url":"https://api.github.com/users/csm/gists{/gist_id}","starred_url":"https://api.github.com/users/csm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/csm/subscriptions","organizations_url":"https://api.github.com/users/csm/orgs","repos_url":"https://api.github.com/users/csm/repos","events_url":"https://api.github.com/users/csm/events{/privacy}","received_events_url":"https://api.github.com/users/csm/received_events","type":"User","site_admin":false},"repo":{"id":65314804,"name":"hitchhiker-tree","full_name":"csm/hitchhiker-tree","owner":{"login":"csm","id":39441,"avatar_url":"https://avatars.githubusercontent.com/u/39441?v=3","gravatar_id":"","url":"https://api.github.com/users/csm","html_url":"https://github.com/csm","followers_url":"https://api.github.com/users/csm/followers","following_url":"https://api.github.com/users/csm/following{/other_user}","gists_url":"https://api.github.com/users/csm/gists{/gist_id}","starred_url":"https://api.github.com/users/csm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/csm/subscriptions","organizations_url":"https://api.github.com/users/csm/orgs","repos_url":"https://api.github.com/users/csm/repos","events_url":"https://api.github.com/users/csm/events{/privacy}","received_events_url":"https://api.github.com/users/csm/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/csm/hitchhiker-tree","description":"Functional, persistent, off-heap, high performance data structure","fork":true,"url":"https://api.github.com/repos/csm/hitchhiker-tree","forks_url":"https://api.github.com/repos/csm/hitchhiker-tree/forks","keys_url":"https://api.github.com/repos/csm/hitchhiker-tree/keys{/key_id}","collaborators_url":"https://api.github.com/repos/csm/hitchhiker-tree/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/csm/hitchhiker-tree/teams","hooks_url":"https://api.github.com/repos/csm/hitchhiker-tree/hooks","issue_events_url":"https://api.github.com/repos/csm/hitchhiker-tree/issues/events{/number}","events_url":"https://api.github.com/repos/csm/hitchhiker-tree/events","assignees_url":"https://api.github.com/repos/csm/hitchhiker-tree/assignees{/user}","branches_url":"https://api.github.com/repos/csm/hitchhiker-tree/branches{/branch}","tags_url":"https://api.github.com/repos/csm/hitchhiker-tree/tags","blobs_url":"https://api.github.com/repos/csm/hitchhiker-tree/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/csm/hitchhiker-tree/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/csm/hitchhiker-tree/git/refs{/sha}","trees_url":"https://api.github.com/repos/csm/hitchhiker-tree/git/trees{/sha}","statuses_url":"https://api.github.com/repos/csm/hitchhiker-tree/statuses/{sha}","languages_url":"https://api.github.com/repos/csm/hitchhiker-tree/languages","stargazers_url":"https://api.github.com/repos/csm/hitchhiker-tree/stargazers","contributors_url":"https://api.github.com/repos/csm/hitchhiker-tree/contributors","subscribers_url":"https://api.github.com/repos/csm/hitchhiker-tree/subscribers","subscription_url":"https://api.github.com/repos/csm/hitchhiker-tree/subscription","commits_url":"https://api.github.com/repos/csm/hitchhiker-tree/commits{/sha}","git_commits_url":"https://api.github.com/repos/csm/hitchhiker-tree/git/commits{/sha}","comments_url":"https://api.github.com/repos/csm/hitchhiker-tree/comments{/number}","issue_comment_url":"https://api.github.com/repos/csm/hitchhiker-tree/issues/comments{/number}","contents_url":"https://api.github.com/repos/csm/hitchhiker-tree/contents/{+path}","compare_url":"https://api.github.com/repos/csm/hitchhiker-tree/compare/{base}...{head}","merges_url":"https://api.github.com/repos/csm/hitchhiker-tree/merges","archive_url":"https://api.github.com/repos/csm/hitchhiker-tree/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/csm/hitchhiker-tree/downloads","issues_url":"https://api.github.com/repos/csm/hitchhiker-tree/issues{/number}","pulls_url":"https://api.github.com/repos/csm/hitchhiker-tree/pulls{/number}","milestones_url":"https://api.github.com/repos/csm/hitchhiker-tree/milestones{/number}","notifications_url":"https://api.github.com/repos/csm/hitchhiker-tree/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/csm/hitchhiker-tree/labels{/name}","releases_url":"https://api.github.com/repos/csm/hitchhiker-tree/releases{/id}","deployments_url":"https://api.github.com/repos/csm/hitchhiker-tree/deployments","created_at":"2016-08-09T17:30:08Z","updated_at":"2016-08-09T17:30:10Z","pushed_at":"2016-09-02T16:11:07Z","git_url":"git://github.com/csm/hitchhiker-tree.git","ssh_url":"git@github.com:csm/hitchhiker-tree.git","clone_url":"https://github.com/csm/hitchhiker-tree.git","svn_url":"https://github.com/csm/hitchhiker-tree","homepage":"","size":221,"stargazers_count":0,"watchers_count":0,"language":"Clojure","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"datacrypt-project:master","ref":"master","sha":"27ebb9b369512e8db91b7ac41a7c43ce03a09299","user":{"login":"datacrypt-project","id":18667846,"avatar_url":"https://avatars.githubusercontent.com/u/18667846?v=3","gravatar_id":"","url":"https://api.github.com/users/datacrypt-project","html_url":"https://github.com/datacrypt-project","followers_url":"https://api.github.com/users/datacrypt-project/followers","following_url":"https://api.github.com/users/datacrypt-project/following{/other_user}","gists_url":"https://api.github.com/users/datacrypt-project/gists{/gist_id}","starred_url":"https://api.github.com/users/datacrypt-project/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datacrypt-project/subscriptions","organizations_url":"https://api.github.com/users/datacrypt-project/orgs","repos_url":"https://api.github.com/users/datacrypt-project/repos","events_url":"https://api.github.com/users/datacrypt-project/events{/privacy}","received_events_url":"https://api.github.com/users/datacrypt-project/received_events","type":"Organization","site_admin":false},"repo":{"id":52108968,"name":"hitchhiker-tree","full_name":"datacrypt-project/hitchhiker-tree","owner":{"login":"datacrypt-project","id":18667846,"avatar_url":"https://avatars.githubusercontent.com/u/18667846?v=3","gravatar_id":"","url":"https://api.github.com/users/datacrypt-project","html_url":"https://github.com/datacrypt-project","followers_url":"https://api.github.com/users/datacrypt-project/followers","following_url":"https://api.github.com/users/datacrypt-project/following{/other_user}","gists_url":"https://api.github.com/users/datacrypt-project/gists{/gist_id}","starred_url":"https://api.github.com/users/datacrypt-project/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datacrypt-project/subscriptions","organizations_url":"https://api.github.com/users/datacrypt-project/orgs","repos_url":"https://api.github.com/users/datacrypt-project/repos","events_url":"https://api.github.com/users/datacrypt-project/events{/privacy}","received_events_url":"https://api.github.com/users/datacrypt-project/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/datacrypt-project/hitchhiker-tree","description":"Functional, persistent, off-heap, high performance data structure","fork":false,"url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree","forks_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/forks","keys_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/keys{/key_id}","collaborators_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/teams","hooks_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/hooks","issue_events_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/events{/number}","events_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/events","assignees_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/assignees{/user}","branches_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/branches{/branch}","tags_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/tags","blobs_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/git/refs{/sha}","trees_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/git/trees{/sha}","statuses_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/statuses/{sha}","languages_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/languages","stargazers_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/stargazers","contributors_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/contributors","subscribers_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/subscribers","subscription_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/subscription","commits_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/commits{/sha}","git_commits_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/git/commits{/sha}","comments_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/comments{/number}","issue_comment_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/comments{/number}","contents_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/contents/{+path}","compare_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/compare/{base}...{head}","merges_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/merges","archive_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/downloads","issues_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues{/number}","pulls_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls{/number}","milestones_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/milestones{/number}","notifications_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/labels{/name}","releases_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/releases{/id}","deployments_url":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/deployments","created_at":"2016-02-19T18:49:16Z","updated_at":"2016-09-02T11:01:28Z","pushed_at":"2016-09-02T16:24:30Z","git_url":"git://github.com/datacrypt-project/hitchhiker-tree.git","ssh_url":"git@github.com:datacrypt-project/hitchhiker-tree.git","clone_url":"https://github.com/datacrypt-project/hitchhiker-tree.git","svn_url":"https://github.com/datacrypt-project/hitchhiker-tree","homepage":"","size":223,"stargazers_count":642,"watchers_count":642,"language":"Clojure","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":35,"mirror_url":null,"open_issues_count":11,"forks":35,"open_issues":11,"watchers":642,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23"},"html":{"href":"https://github.com/datacrypt-project/hitchhiker-tree/pull/23"},"issue":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/23"},"comments":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/issues/23/comments"},"review_comments":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23/comments"},"review_comment":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/pulls/23/commits"},"statuses":{"href":"https://api.github.com/repos/datacrypt-project/hitchhiker-tree/statuses/261e764878971214b5f08d7c266a764ab8b22546"}}}}
{"actor":{"display_login":"arichiardi"}}