{"action":"created","comment":{"url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments/79213353","id":79213353,"diff_hunk":"@@ -30,15 +32,78 @@\n     (catch Exception e\n       (throw+ {:kind ::parse-error} e \"Error parsing command\"))))\n \n+(defn sanitize-certname\n+  \"Replace any underscores and filename forbidden characters found in `certname`\n+  with dashes.\"\n+  [certname]\n+  (let [forbidden-chars (conj constants/filename-forbidden-characters\n+                              \"_\")]\n+    (str/replace\n+      certname\n+      (re-pattern (str \"(\"\n+                       (->> forbidden-chars\n+                           (map #(format \"\\\\Q%s\\\\E\" %)) ; escape regex chars\n+                           (str/join \"|\"))\n+                       \")\"))\n+      \"-\")))\n+\n+(defn max-certname-length\n+  \"Given the received-at time, command and command version for a metadata\n+  string, returns the maximum allowable length (in bytes) of a certname in that\n+  string. Note that this length is only achievable if the certname does not\n+  need to be sanitized, since sanitized certnames always have a SHA1 hash\n+  appended.\"\n+  [received command version]\n+  (let [time-length (-> received tcoerce/to-long str utf8-length)\n+        command-length (utf8-length command)\n+        version-length (utf8-length (str version))\n+        field-separators 3]\n+    (- 255 ; overall filename length limit\n+       time-length command-length version-length\n+       3 ; number of field separators (underscores)\n+       (utf8-length \".json\"))))\n+\n+(defn truncated-certname-length\n+  \"Given the received-at time, command, and command version that will be in\n+  a metadata string, returns the length (in bytes) to truncate certnames to when\n+  they are longer than the `max-certname-length` for the string or they contain\n+  characters that must be sanitized. This is less than the string's\n+  `max-certname-length` to leave space for the SHA1 hash.\"\n+  [received command version]\n+  (- (max-certname-length received command version)\n+     1 ; for the additional field separator between certname and hash\n+     (utf8-length (kitchensink/utf8-string->sha1 \"\"))))\n+\n+(defn embeddable-certname\n+  \"Takes all the components of a metadata string, and returns a version of the\n+  certname that's safe to embed in the metadata string. It will be sanitized to\n+  replace any illegal filesystem characters and underscores with dashes, and\n+  will be truncated if the original version will cause the metadata string to\n+  exceed 255 characters.\"\n+  [received command version certname]\n+  (let [cn-length (utf8-length certname)\n+        trunc-length (truncated-certname-length received command version)\n+        sanitized-certname (sanitize-certname certname)]\n+    (if (or (> cn-length (max-certname-length received command version))\n+            (and (not= certname sanitized-certname)\n+                 (> cn-length trunc-length)))\n+      (utf8-truncate sanitized-certname trunc-length)\n+      sanitized-certname)))\n+\n (defn metadata-str [received command version certname]\n-  (format \"%d_%s_%d_%s.json\"\n-          (tcoerce/to-long received) command version certname))\n+  (let [certname (or certname \"unknown-host\")\n+        recvd-long (tcoerce/to-long received)\n+        safe-certname (embeddable-certname received command version certname)]\n+    (if (= safe-certname certname)\n+      (format \"%d_%s_%d_%s.json\" recvd-long command version safe-certname)\n+      (let [name-hash (kitchensink/utf8-string->sha1 certname)]\n+        (format \"%d_%s_%d_%s_%s.json\"\n+                recvd-long command version safe-certname name-hash)))))\n \n (defn- metadata-rx [valid-commands]\n   (re-pattern (str\n                \"([0-9]+)_(\"\n-               (str/join \"|\" (map #(format \"(?:%s)\" (re-quote-replacement %))\n-                                  valid-commands))\n+               (str/join \"|\" (map #(format \"\\\\Q%s\\\\E\" %) valid-commands))","path":"src/puppetlabs/puppetdb/queue.clj","position":103,"original_position":103,"commit_id":"dc4cebdb1485c31cf1a632fbce71fe81d9ba65d6","original_commit_id":"dc4cebdb1485c31cf1a632fbce71fe81d9ba65d6","user":{"login":"rbrw","id":10120698,"avatar_url":"https://avatars.githubusercontent.com/u/10120698?v=3","gravatar_id":"","url":"https://api.github.com/users/rbrw","html_url":"https://github.com/rbrw","followers_url":"https://api.github.com/users/rbrw/followers","following_url":"https://api.github.com/users/rbrw/following{/other_user}","gists_url":"https://api.github.com/users/rbrw/gists{/gist_id}","starred_url":"https://api.github.com/users/rbrw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rbrw/subscriptions","organizations_url":"https://api.github.com/users/rbrw/orgs","repos_url":"https://api.github.com/users/rbrw/repos","events_url":"https://api.github.com/users/rbrw/events{/privacy}","received_events_url":"https://api.github.com/users/rbrw/received_events","type":"User","site_admin":false},"body":"Oh, it may not matter if we know that the input here is trusted, and it may well be (or we can guard elsewhere), but just happened to notice it while thinking through the behavior.","created_at":"2016-09-16T17:22:08Z","updated_at":"2016-09-16T17:22:08Z","html_url":"https://github.com/puppetlabs/puppetdb/pull/2065#discussion_r79213353","pull_request_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065","_links":{"self":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments/79213353"},"html":{"href":"https://github.com/puppetlabs/puppetdb/pull/2065#discussion_r79213353"},"pull_request":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065"}}},"pull_request":{"url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065","id":85320729,"html_url":"https://github.com/puppetlabs/puppetdb/pull/2065","diff_url":"https://github.com/puppetlabs/puppetdb/pull/2065.diff","patch_url":"https://github.com/puppetlabs/puppetdb/pull/2065.patch","issue_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/2065","number":2065,"state":"open","locked":false,"title":"(PDB-3039) Safely embed certnames in queue & DLO filenames","user":{"login":"aperiodic","id":115510,"avatar_url":"https://avatars.githubusercontent.com/u/115510?v=3","gravatar_id":"","url":"https://api.github.com/users/aperiodic","html_url":"https://github.com/aperiodic","followers_url":"https://api.github.com/users/aperiodic/followers","following_url":"https://api.github.com/users/aperiodic/following{/other_user}","gists_url":"https://api.github.com/users/aperiodic/gists{/gist_id}","starred_url":"https://api.github.com/users/aperiodic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aperiodic/subscriptions","organizations_url":"https://api.github.com/users/aperiodic/orgs","repos_url":"https://api.github.com/users/aperiodic/repos","events_url":"https://api.github.com/users/aperiodic/events{/privacy}","received_events_url":"https://api.github.com/users/aperiodic/received_events","type":"User","site_admin":false},"body":"Change `puppetdb.queue/metadata-str` to replace illegal filename\r\ncharacters and underscores in the certname it's given, truncate the\r\ncertname if its length would cause the metadata string to exceed 255\r\ncharacters, and add the SHA1 hash of the original certname if the\r\ncertname in the metadata string differs in any way (via character\r\nreplacement or truncation) from the original certname.\r\n\r\nChange the *-filename functions in `puppetlabs.command.dlo` to truncate\r\njust the metadata portions of the filenames if they would cause the\r\nfilenames to exceed 255 bytes in length. Truncating the metadata portion\r\nis acceptable since all these filenames contain stockpile IDs, which\r\nensures they'll be unique.\r\n\r\nSee the individual commits & messages for more details.","created_at":"2016-09-14T23:01:08Z","updated_at":"2016-09-16T17:22:08Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9c9d174acd7b1c9953487c889b2842e3a28b1e9b","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065/commits","review_comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065/comments","review_comment_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/2065/comments","statuses_url":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/dc4cebdb1485c31cf1a632fbce71fe81d9ba65d6","head":{"label":"aperiodic:pdb-3039-safe-certnames-in-filenames","ref":"pdb-3039-safe-certnames-in-filenames","sha":"dc4cebdb1485c31cf1a632fbce71fe81d9ba65d6","user":{"login":"aperiodic","id":115510,"avatar_url":"https://avatars.githubusercontent.com/u/115510?v=3","gravatar_id":"","url":"https://api.github.com/users/aperiodic","html_url":"https://github.com/aperiodic","followers_url":"https://api.github.com/users/aperiodic/followers","following_url":"https://api.github.com/users/aperiodic/following{/other_user}","gists_url":"https://api.github.com/users/aperiodic/gists{/gist_id}","starred_url":"https://api.github.com/users/aperiodic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aperiodic/subscriptions","organizations_url":"https://api.github.com/users/aperiodic/orgs","repos_url":"https://api.github.com/users/aperiodic/repos","events_url":"https://api.github.com/users/aperiodic/events{/privacy}","received_events_url":"https://api.github.com/users/aperiodic/received_events","type":"User","site_admin":false},"repo":{"id":67833014,"name":"puppetdb","full_name":"aperiodic/puppetdb","owner":{"login":"aperiodic","id":115510,"avatar_url":"https://avatars.githubusercontent.com/u/115510?v=3","gravatar_id":"","url":"https://api.github.com/users/aperiodic","html_url":"https://github.com/aperiodic","followers_url":"https://api.github.com/users/aperiodic/followers","following_url":"https://api.github.com/users/aperiodic/following{/other_user}","gists_url":"https://api.github.com/users/aperiodic/gists{/gist_id}","starred_url":"https://api.github.com/users/aperiodic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aperiodic/subscriptions","organizations_url":"https://api.github.com/users/aperiodic/orgs","repos_url":"https://api.github.com/users/aperiodic/repos","events_url":"https://api.github.com/users/aperiodic/events{/privacy}","received_events_url":"https://api.github.com/users/aperiodic/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/aperiodic/puppetdb","description":"Centralized Puppet Storage","fork":true,"url":"https://api.github.com/repos/aperiodic/puppetdb","forks_url":"https://api.github.com/repos/aperiodic/puppetdb/forks","keys_url":"https://api.github.com/repos/aperiodic/puppetdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aperiodic/puppetdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aperiodic/puppetdb/teams","hooks_url":"https://api.github.com/repos/aperiodic/puppetdb/hooks","issue_events_url":"https://api.github.com/repos/aperiodic/puppetdb/issues/events{/number}","events_url":"https://api.github.com/repos/aperiodic/puppetdb/events","assignees_url":"https://api.github.com/repos/aperiodic/puppetdb/assignees{/user}","branches_url":"https://api.github.com/repos/aperiodic/puppetdb/branches{/branch}","tags_url":"https://api.github.com/repos/aperiodic/puppetdb/tags","blobs_url":"https://api.github.com/repos/aperiodic/puppetdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aperiodic/puppetdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aperiodic/puppetdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/aperiodic/puppetdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aperiodic/puppetdb/statuses/{sha}","languages_url":"https://api.github.com/repos/aperiodic/puppetdb/languages","stargazers_url":"https://api.github.com/repos/aperiodic/puppetdb/stargazers","contributors_url":"https://api.github.com/repos/aperiodic/puppetdb/contributors","subscribers_url":"https://api.github.com/repos/aperiodic/puppetdb/subscribers","subscription_url":"https://api.github.com/repos/aperiodic/puppetdb/subscription","commits_url":"https://api.github.com/repos/aperiodic/puppetdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/aperiodic/puppetdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/aperiodic/puppetdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/aperiodic/puppetdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/aperiodic/puppetdb/contents/{+path}","compare_url":"https://api.github.com/repos/aperiodic/puppetdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aperiodic/puppetdb/merges","archive_url":"https://api.github.com/repos/aperiodic/puppetdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aperiodic/puppetdb/downloads","issues_url":"https://api.github.com/repos/aperiodic/puppetdb/issues{/number}","pulls_url":"https://api.github.com/repos/aperiodic/puppetdb/pulls{/number}","milestones_url":"https://api.github.com/repos/aperiodic/puppetdb/milestones{/number}","notifications_url":"https://api.github.com/repos/aperiodic/puppetdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aperiodic/puppetdb/labels{/name}","releases_url":"https://api.github.com/repos/aperiodic/puppetdb/releases{/id}","deployments_url":"https://api.github.com/repos/aperiodic/puppetdb/deployments","created_at":"2016-09-09T21:11:26Z","updated_at":"2016-09-09T21:11:29Z","pushed_at":"2016-09-16T00:21:53Z","git_url":"git://github.com/aperiodic/puppetdb.git","ssh_url":"git@github.com:aperiodic/puppetdb.git","clone_url":"https://github.com/aperiodic/puppetdb.git","svn_url":"https://github.com/aperiodic/puppetdb","homepage":"http://docs.puppetlabs.com/puppetdb","size":16689,"stargazers_count":0,"watchers_count":0,"language":"Clojure","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"puppetlabs:cmd-proc-refactor","ref":"cmd-proc-refactor","sha":"fd9cc98f51f9649a0aa5d64f7b988df7458e9c0a","user":{"login":"puppetlabs","id":234268,"avatar_url":"https://avatars.githubusercontent.com/u/234268?v=3","gravatar_id":"","url":"https://api.github.com/users/puppetlabs","html_url":"https://github.com/puppetlabs","followers_url":"https://api.github.com/users/puppetlabs/followers","following_url":"https://api.github.com/users/puppetlabs/following{/other_user}","gists_url":"https://api.github.com/users/puppetlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/puppetlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/puppetlabs/subscriptions","organizations_url":"https://api.github.com/users/puppetlabs/orgs","repos_url":"https://api.github.com/users/puppetlabs/repos","events_url":"https://api.github.com/users/puppetlabs/events{/privacy}","received_events_url":"https://api.github.com/users/puppetlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":2193032,"name":"puppetdb","full_name":"puppetlabs/puppetdb","owner":{"login":"puppetlabs","id":234268,"avatar_url":"https://avatars.githubusercontent.com/u/234268?v=3","gravatar_id":"","url":"https://api.github.com/users/puppetlabs","html_url":"https://github.com/puppetlabs","followers_url":"https://api.github.com/users/puppetlabs/followers","following_url":"https://api.github.com/users/puppetlabs/following{/other_user}","gists_url":"https://api.github.com/users/puppetlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/puppetlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/puppetlabs/subscriptions","organizations_url":"https://api.github.com/users/puppetlabs/orgs","repos_url":"https://api.github.com/users/puppetlabs/repos","events_url":"https://api.github.com/users/puppetlabs/events{/privacy}","received_events_url":"https://api.github.com/users/puppetlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/puppetlabs/puppetdb","description":"Centralized Puppet Storage","fork":false,"url":"https://api.github.com/repos/puppetlabs/puppetdb","forks_url":"https://api.github.com/repos/puppetlabs/puppetdb/forks","keys_url":"https://api.github.com/repos/puppetlabs/puppetdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/puppetlabs/puppetdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/puppetlabs/puppetdb/teams","hooks_url":"https://api.github.com/repos/puppetlabs/puppetdb/hooks","issue_events_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/events{/number}","events_url":"https://api.github.com/repos/puppetlabs/puppetdb/events","assignees_url":"https://api.github.com/repos/puppetlabs/puppetdb/assignees{/user}","branches_url":"https://api.github.com/repos/puppetlabs/puppetdb/branches{/branch}","tags_url":"https://api.github.com/repos/puppetlabs/puppetdb/tags","blobs_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/{sha}","languages_url":"https://api.github.com/repos/puppetlabs/puppetdb/languages","stargazers_url":"https://api.github.com/repos/puppetlabs/puppetdb/stargazers","contributors_url":"https://api.github.com/repos/puppetlabs/puppetdb/contributors","subscribers_url":"https://api.github.com/repos/puppetlabs/puppetdb/subscribers","subscription_url":"https://api.github.com/repos/puppetlabs/puppetdb/subscription","commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/puppetlabs/puppetdb/contents/{+path}","compare_url":"https://api.github.com/repos/puppetlabs/puppetdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/puppetlabs/puppetdb/merges","archive_url":"https://api.github.com/repos/puppetlabs/puppetdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/puppetlabs/puppetdb/downloads","issues_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues{/number}","pulls_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls{/number}","milestones_url":"https://api.github.com/repos/puppetlabs/puppetdb/milestones{/number}","notifications_url":"https://api.github.com/repos/puppetlabs/puppetdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/puppetlabs/puppetdb/labels{/name}","releases_url":"https://api.github.com/repos/puppetlabs/puppetdb/releases{/id}","deployments_url":"https://api.github.com/repos/puppetlabs/puppetdb/deployments","created_at":"2011-08-11T18:54:46Z","updated_at":"2016-09-13T10:21:50Z","pushed_at":"2016-09-15T23:53:16Z","git_url":"git://github.com/puppetlabs/puppetdb.git","ssh_url":"git@github.com:puppetlabs/puppetdb.git","clone_url":"https://github.com/puppetlabs/puppetdb.git","svn_url":"https://github.com/puppetlabs/puppetdb","homepage":"http://docs.puppetlabs.com/puppetdb","size":20376,"stargazers_count":194,"watchers_count":194,"language":"Clojure","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":161,"mirror_url":null,"open_issues_count":6,"forks":161,"open_issues":6,"watchers":194,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065"},"html":{"href":"https://github.com/puppetlabs/puppetdb/pull/2065"},"issue":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/issues/2065"},"comments":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/issues/2065/comments"},"review_comments":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065/comments"},"review_comment":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/2065/commits"},"statuses":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/dc4cebdb1485c31cf1a632fbce71fe81d9ba65d6"}}}}
{"actor":{"display_login":"rbrw"}}