{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/activity-streams/pulls/comments/56743372","id":56743372,"diff_hunk":"@@ -0,0 +1,239 @@\n+/* globals Task */\n+\"use strict\";\n+\n+const {Cu} = require(\"chrome\");\n+const ss = require(\"sdk/simple-storage\");\n+const {setTimeout, clearTimeout} = require(\"sdk/timers\");\n+const EMBEDLY_ENDPOINT = require(\"sdk/simple-prefs\").prefs.embedlyEndpoint;\n+\n+const EMBEDLY_PROXY_MAX_LINKS = 25; // number of links embedly proxy accepts per request\n+const CLEAR_CACHE_TIMEOUT = 86400000; // 24 hours to clear cache\n+const REMOVE_KEYS = [\"auth\", \"password\", \"username\", \"pass\", \"user\"];\n+const ALLOWED_PROTOCOLS = new Set([\"http:\", \"https:\"]);\n+const DISALLOWED_HOSTS = new Set([\"localhost\", \"127.0.0.1\", \"0.0.0.0\"]);\n+const URL_FILTERS = [\n+  (item) => !!item.url,\n+  (item) => !!(new URL(item.url)),\n+  (item) => ALLOWED_PROTOCOLS.has(new URL(item.url).protocol),\n+  (item) => !DISALLOWED_HOSTS.has(new URL(item.url).hostname)\n+];\n+\n+Cu.importGlobalProperties([\"fetch\"]);\n+Cu.importGlobalProperties([\"URL\"]);\n+Cu.import(\"resource://gre/modules/Task.jsm\");\n+\n+function PreviewProvider() {\n+  this.init();\n+}\n+\n+PreviewProvider.prototype = {\n+  _timeoutID: null,\n+\n+  /**\n+    * Clean up cache on addon bootup\n+    */\n+  cleanUpCache: Task.async(function*() {\n+    let currentTime = Date.now();\n+    Object.keys(ss.storage.embedlyData).forEach(item => {\n+      if (currentTime - ss.storage.embedlyData[item].accessTime > CLEAR_CACHE_TIMEOUT) {\n+        delete ss.storage.embedlyData[item];\n+      }\n+    });\n+    this._cacheTimeout();\n+  }),\n+\n+  /**\n+    * Set up the timer for the cache to be cleaned every 24 hours\n+    */\n+  _cacheTimeout() {\n+    this._timeoutID = setTimeout(() => {\n+      this.cleanUpCache();\n+    }, CLEAR_CACHE_TIMEOUT);\n+  },\n+\n+  /**\n+    * Filter out sites that do not have acceptable protocols and hosts\n+    */\n+  _URLFilter(definition) {\n+    return function(item) {\n+      let result = true;\n+      definition.forEach((test) => {\n+        if (!test(item)) {\n+          result = false;\n+        }\n+      });\n+      return result;\n+    };\n+  },\n+\n+  /**\n+    * Santize the URL to remove any unwanted or sensitive information about the link\n+    */\n+  _sanitizeURL(site) {\n+    if (!site) {\n+      return \"\";\n+    }\n+\n+    let queryParams = new Map();\n+    let safeQueryParam = \"\";\n+    let url = new URL(site);\n+\n+    if (url.search.length === 0) {\n+      return url.toString();\n+    }\n+    // extract and parse the query parameters, if any\n+    url.search.slice(1, url.search.length);\n+    let paramArray = url.search.slice(1, url.search.length).split(\"&\");\n+    paramArray.forEach(item => {\n+      let tmp = item.split(\"=\");\n+      queryParams.set(tmp[0], tmp[1]);\n+    });\n+    // remove the unwanted parameters and update the query string\n+    REMOVE_KEYS.forEach(key => queryParams.delete(key));\n+    queryParams.forEach((val, key) => {\n+      if (val.length !== 0 && key.length !== 0) {\n+        safeQueryParam = safeQueryParam + key + \"=\" + val + \"&\";\n+      }\n+    });\n+    url.search = safeQueryParam.slice(0, safeQueryParam.length - 1);\n+    url.hash = \"\";\n+\n+    // remove extra slashes in pathname and construct back a safe pathname\n+    if (url.pathname) {\n+      let pathItems = url.pathname.split(\"/\");\n+      let safePathItems = pathItems.filter(item => item.length !== 0);\n+      let safePath = \"/\";\n+      url.pathname = safePath + safePathItems.join(\"/\");\n+    }\n+\n+    // if the url doesn't have a protocol, use https\n+    if (!url.protocol) {\n+      url.protocol = \"https:\";\n+    }\n+\n+    return url.toString();\n+  },\n+\n+  /**\n+    * Create a key based on a URL in order to dedupe sites\n+    */\n+  _createKey(url) {\n+    url = new URL(url);\n+    let key = url.host.replace(/www\\.?/, \"\");\n+    key = key + url.pathname;\n+    return key;\n+  },\n+\n+  /**\n+    * Canonicalize urls by deduping them, then sanitizing them\n+    */\n+  _dedupeURLs(sites) {\n+    let dedupedSites = new Map();\n+    sites.forEach(site => {\n+      let urlkey = this._createKey(site.url);\n+      if (!dedupedSites.has(urlkey)) {\n+        dedupedSites.set(urlkey, site);\n+      } else {\n+        // if two sites come in with 'http' and 'https', pick 'https'\n+        if (!dedupedSites.get(urlkey).url.contains(\"https\")) {\n+          dedupedSites.delete(urlkey);\n+          dedupedSites.set(urlkey, site);\n+        }\n+      }\n+    });\n+    let freshSites = [];\n+    dedupedSites.forEach((site) => {\n+      // sanitize and normalize each site\n+      site.url = this._sanitizeURL(site.url);\n+      freshSites.push(site);\n+    });\n+    return freshSites;\n+  },\n+\n+  /**\n+    * Filter cached links out, and store their embedly data\n+    */\n+  getCachedLinks(links) {","path":"lib/PreviewProvider.js","position":156,"original_position":156,"commit_id":"740cfe65947dea9625265e781e0491e20cb63b53","original_commit_id":"740cfe65947dea9625265e781e0491e20cb63b53","user":{"login":"oyiptong","id":9365,"avatar_url":"https://avatars.githubusercontent.com/u/9365?v=3","gravatar_id":"","url":"https://api.github.com/users/oyiptong","html_url":"https://github.com/oyiptong","followers_url":"https://api.github.com/users/oyiptong/followers","following_url":"https://api.github.com/users/oyiptong/following{/other_user}","gists_url":"https://api.github.com/users/oyiptong/gists{/gist_id}","starred_url":"https://api.github.com/users/oyiptong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oyiptong/subscriptions","organizations_url":"https://api.github.com/users/oyiptong/orgs","repos_url":"https://api.github.com/users/oyiptong/repos","events_url":"https://api.github.com/users/oyiptong/events{/privacy}","received_events_url":"https://api.github.com/users/oyiptong/received_events","type":"User","site_admin":false},"body":"The function will also filter out the URLs for which we have no preview","created_at":"2016-03-19T03:32:23Z","updated_at":"2016-03-19T03:32:23Z","html_url":"https://github.com/mozilla/activity-streams/pull/304#discussion_r56743372","pull_request_url":"https://api.github.com/repos/mozilla/activity-streams/pulls/304","_links":{"self":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/comments/56743372"},"html":{"href":"https://github.com/mozilla/activity-streams/pull/304#discussion_r56743372"},"pull_request":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/304"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/activity-streams/pulls/304","id":62945965,"html_url":"https://github.com/mozilla/activity-streams/pull/304","diff_url":"https://github.com/mozilla/activity-streams/pull/304.diff","patch_url":"https://github.com/mozilla/activity-streams/pull/304.patch","issue_url":"https://api.github.com/repos/mozilla/activity-streams/issues/304","number":304,"state":"open","locked":false,"title":"feat(addon): #258 Implement client in the addon","user":{"login":"sarracini","id":7219526,"avatar_url":"https://avatars.githubusercontent.com/u/7219526?v=3","gravatar_id":"","url":"https://api.github.com/users/sarracini","html_url":"https://github.com/sarracini","followers_url":"https://api.github.com/users/sarracini/followers","following_url":"https://api.github.com/users/sarracini/following{/other_user}","gists_url":"https://api.github.com/users/sarracini/gists{/gist_id}","starred_url":"https://api.github.com/users/sarracini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sarracini/subscriptions","organizations_url":"https://api.github.com/users/sarracini/orgs","repos_url":"https://api.github.com/users/sarracini/repos","events_url":"https://api.github.com/users/sarracini/events{/privacy}","received_events_url":"https://api.github.com/users/sarracini/received_events","type":"User","site_admin":false},"body":"Things this does:\r\n1. checks a local disk-based cache for an existing URLs and obtain embedly data using simple-storage (on addon side)\r\n2. for urls which cause a cache miss, formulates POST query to send to embedly. Upon receiving a response, stores embedly data in cache + a timestamp\r\n3. return results that have embedly data to content\r\n4. periodically cleans up the cache for items > 24H\r\n5. on addon boot up, and every 6 hours, asynchronously primes the cache with queries that a normal content page load would \r\n6. removes content-side embedly middleware which made requests every single time (yay!)\r\n\r\nAs we discussed, this only sends down things that are cached and so it doesn't block on embedly returning the data. Instead, it will add it to the cache and the next time a request for that link comes up, it will have the stored data. The fetching is done asynchronously so we don't block on the embedly server for getting the cached links down. Since the server handles 25 urls per request, a request queue is made with 25 links at a time, and a new async task to request embedly data is spawned for each bundle of 25.\r\n\r\nThings to do:\r\n1. Santize, dedupe, filter incoming URLS\r\n\r\nReview can be started on the parts that are done while I work on the things that are left to do\r\n@oyiptong @k88hudson \r\n\r\nFixes #258 ","created_at":"2016-03-15T17:32:23Z","updated_at":"2016-03-19T03:32:23Z","closed_at":null,"merged_at":null,"merge_commit_sha":"51f39f374404531950e4c54a0477ca912d72a02e","assignee":{"login":"sarracini","id":7219526,"avatar_url":"https://avatars.githubusercontent.com/u/7219526?v=3","gravatar_id":"","url":"https://api.github.com/users/sarracini","html_url":"https://github.com/sarracini","followers_url":"https://api.github.com/users/sarracini/followers","following_url":"https://api.github.com/users/sarracini/following{/other_user}","gists_url":"https://api.github.com/users/sarracini/gists{/gist_id}","starred_url":"https://api.github.com/users/sarracini/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sarracini/subscriptions","organizations_url":"https://api.github.com/users/sarracini/orgs","repos_url":"https://api.github.com/users/sarracini/repos","events_url":"https://api.github.com/users/sarracini/events{/privacy}","received_events_url":"https://api.github.com/users/sarracini/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/activity-streams/pulls/304/commits","review_comments_url":"https://api.github.com/repos/mozilla/activity-streams/pulls/304/comments","review_comment_url":"https://api.github.com/repos/mozilla/activity-streams/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/activity-streams/issues/304/comments","statuses_url":"https://api.github.com/repos/mozilla/activity-streams/statuses/740cfe65947dea9625265e781e0491e20cb63b53","head":{"label":"mozilla:embedly_caching","ref":"embedly_caching","sha":"740cfe65947dea9625265e781e0491e20cb63b53","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":50695589,"name":"activity-streams","full_name":"mozilla/activity-streams","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/activity-streams","description":" MVP prototype add-on for activity streams","fork":false,"url":"https://api.github.com/repos/mozilla/activity-streams","forks_url":"https://api.github.com/repos/mozilla/activity-streams/forks","keys_url":"https://api.github.com/repos/mozilla/activity-streams/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/activity-streams/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/activity-streams/teams","hooks_url":"https://api.github.com/repos/mozilla/activity-streams/hooks","issue_events_url":"https://api.github.com/repos/mozilla/activity-streams/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/activity-streams/events","assignees_url":"https://api.github.com/repos/mozilla/activity-streams/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/activity-streams/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/activity-streams/tags","blobs_url":"https://api.github.com/repos/mozilla/activity-streams/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/activity-streams/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/activity-streams/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/activity-streams/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/activity-streams/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/activity-streams/languages","stargazers_url":"https://api.github.com/repos/mozilla/activity-streams/stargazers","contributors_url":"https://api.github.com/repos/mozilla/activity-streams/contributors","subscribers_url":"https://api.github.com/repos/mozilla/activity-streams/subscribers","subscription_url":"https://api.github.com/repos/mozilla/activity-streams/subscription","commits_url":"https://api.github.com/repos/mozilla/activity-streams/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/activity-streams/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/activity-streams/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/activity-streams/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/activity-streams/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/activity-streams/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/activity-streams/merges","archive_url":"https://api.github.com/repos/mozilla/activity-streams/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/activity-streams/downloads","issues_url":"https://api.github.com/repos/mozilla/activity-streams/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/activity-streams/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/activity-streams/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/activity-streams/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/activity-streams/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/activity-streams/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/activity-streams/deployments","created_at":"2016-01-29T22:28:26Z","updated_at":"2016-03-18T22:26:10Z","pushed_at":"2016-03-18T23:37:13Z","git_url":"git://github.com/mozilla/activity-streams.git","ssh_url":"git@github.com:mozilla/activity-streams.git","clone_url":"https://github.com/mozilla/activity-streams.git","svn_url":"https://github.com/mozilla/activity-streams","homepage":"","size":528,"stargazers_count":12,"watchers_count":12,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":86,"forks":6,"open_issues":86,"watchers":12,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"170fdf91a02ed137fecfcd74c4f47e1457a4ac21","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":50695589,"name":"activity-streams","full_name":"mozilla/activity-streams","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/activity-streams","description":" MVP prototype add-on for activity streams","fork":false,"url":"https://api.github.com/repos/mozilla/activity-streams","forks_url":"https://api.github.com/repos/mozilla/activity-streams/forks","keys_url":"https://api.github.com/repos/mozilla/activity-streams/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/activity-streams/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/activity-streams/teams","hooks_url":"https://api.github.com/repos/mozilla/activity-streams/hooks","issue_events_url":"https://api.github.com/repos/mozilla/activity-streams/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/activity-streams/events","assignees_url":"https://api.github.com/repos/mozilla/activity-streams/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/activity-streams/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/activity-streams/tags","blobs_url":"https://api.github.com/repos/mozilla/activity-streams/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/activity-streams/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/activity-streams/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/activity-streams/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/activity-streams/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/activity-streams/languages","stargazers_url":"https://api.github.com/repos/mozilla/activity-streams/stargazers","contributors_url":"https://api.github.com/repos/mozilla/activity-streams/contributors","subscribers_url":"https://api.github.com/repos/mozilla/activity-streams/subscribers","subscription_url":"https://api.github.com/repos/mozilla/activity-streams/subscription","commits_url":"https://api.github.com/repos/mozilla/activity-streams/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/activity-streams/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/activity-streams/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/activity-streams/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/activity-streams/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/activity-streams/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/activity-streams/merges","archive_url":"https://api.github.com/repos/mozilla/activity-streams/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/activity-streams/downloads","issues_url":"https://api.github.com/repos/mozilla/activity-streams/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/activity-streams/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/activity-streams/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/activity-streams/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/activity-streams/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/activity-streams/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/activity-streams/deployments","created_at":"2016-01-29T22:28:26Z","updated_at":"2016-03-18T22:26:10Z","pushed_at":"2016-03-18T23:37:13Z","git_url":"git://github.com/mozilla/activity-streams.git","ssh_url":"git@github.com:mozilla/activity-streams.git","clone_url":"https://github.com/mozilla/activity-streams.git","svn_url":"https://github.com/mozilla/activity-streams","homepage":"","size":528,"stargazers_count":12,"watchers_count":12,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":86,"forks":6,"open_issues":86,"watchers":12,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/304"},"html":{"href":"https://github.com/mozilla/activity-streams/pull/304"},"issue":{"href":"https://api.github.com/repos/mozilla/activity-streams/issues/304"},"comments":{"href":"https://api.github.com/repos/mozilla/activity-streams/issues/304/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/304/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/activity-streams/pulls/304/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/activity-streams/statuses/740cfe65947dea9625265e781e0491e20cb63b53"}}}}