{"action":"created","comment":{"url":"https://api.github.com/repos/WorldBrain/cortex/pulls/comments/72315921","id":72315921,"diff_hunk":"@@ -0,0 +1,110 @@\n+import requests\n+import json\n+import re\n+\n+with open(\"facebook_groups_list.txt\", \"r\") as f:\n+    group_ids = []\n+    for line in f:\n+        group_ids.append(line.split()[1])\n+\n+ACCESS_TOKEN = \"\"\n+\n+group_urls = []\n+for group_id in group_ids:\n+    base = \"https://graph.facebook.com/v2.7/\" + group_id + \"/feed/\"\n+    fields = \"message,link,comments{message}\"\n+    url = '{}?fields={}&access_token={}'.format(base, fields, ACCESS_TOKEN)\n+    group_urls.append(url)\n+url = group_urls[2]\n+group_id = group_ids[2]\n+\n+#def pp(content):\n+ #   print(json.dumps(content, indent=1))\n+\n+urls_links = set()\n+urls_messages = set()\n+\n+\n+def scrape_for_urls(url, urls_links, urls_messages):\n+    \"\"\"Gets the json from the request to Facebook Graph API\n+    and then extracts the urls from the post (in case the user deleted the url)\n+    and the message of the post\n+    and also the urls from the comments below the post. It also recursively\n+    takes the next page, since the Graph API has a limit to the results it give\n+    \"\"\"\n+\n+#    print(\"runs\")\n+ #   for group_url in url:\n+  #      content = requests.get(url).json()\n+\n+    print(url)\n+    content = requests.get(url).json()\n+\n+    for entry in content[\"data\"]:\n+        try:\n+            if entry[\"link\"]:\n+                urls_links.add(entry[\"link\"])\n+        except KeyError:\n+            continue\n+\n+    for entry in content[\"data\"]:\n+        try:\n+            for subentree in entry[\"comments\"][\"data\"]:\n+                findit = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', subentree[\"message\"])\n+                for item in findit:\n+                    urls_messages.add(item)\n+        except KeyError:\n+            continue","path":"facebook_crawler/facebook_crawler.py","position":57,"original_position":57,"commit_id":"d4168988c9940990620d3394fec950e02d9f639f","original_commit_id":"d4168988c9940990620d3394fec950e02d9f639f","user":{"login":"lwm","id":1991377,"avatar_url":"https://avatars.githubusercontent.com/u/1991377?v=3","gravatar_id":"","url":"https://api.github.com/users/lwm","html_url":"https://github.com/lwm","followers_url":"https://api.github.com/users/lwm/followers","following_url":"https://api.github.com/users/lwm/following{/other_user}","gists_url":"https://api.github.com/users/lwm/gists{/gist_id}","starred_url":"https://api.github.com/users/lwm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lwm/subscriptions","organizations_url":"https://api.github.com/users/lwm/orgs","repos_url":"https://api.github.com/users/lwm/repos","events_url":"https://api.github.com/users/lwm/events{/privacy}","received_events_url":"https://api.github.com/users/lwm/received_events","type":"User","site_admin":false},"body":"Same note as above.","created_at":"2016-07-26T19:05:46Z","updated_at":"2016-07-26T19:05:46Z","html_url":"https://github.com/WorldBrain/cortex/pull/47#discussion_r72315921","pull_request_url":"https://api.github.com/repos/WorldBrain/cortex/pulls/47","_links":{"self":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/comments/72315921"},"html":{"href":"https://github.com/WorldBrain/cortex/pull/47#discussion_r72315921"},"pull_request":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/47"}}},"pull_request":{"url":"https://api.github.com/repos/WorldBrain/cortex/pulls/47","id":78752290,"html_url":"https://github.com/WorldBrain/cortex/pull/47","diff_url":"https://github.com/WorldBrain/cortex/pull/47.diff","patch_url":"https://github.com/WorldBrain/cortex/pull/47.patch","issue_url":"https://api.github.com/repos/WorldBrain/cortex/issues/47","number":47,"state":"open","locked":false,"title":"Facebook crawler","user":{"login":"chr7stos","id":3687203,"avatar_url":"https://avatars.githubusercontent.com/u/3687203?v=3","gravatar_id":"","url":"https://api.github.com/users/chr7stos","html_url":"https://github.com/chr7stos","followers_url":"https://api.github.com/users/chr7stos/followers","following_url":"https://api.github.com/users/chr7stos/following{/other_user}","gists_url":"https://api.github.com/users/chr7stos/gists{/gist_id}","starred_url":"https://api.github.com/users/chr7stos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chr7stos/subscriptions","organizations_url":"https://api.github.com/users/chr7stos/orgs","repos_url":"https://api.github.com/users/chr7stos/repos","events_url":"https://api.github.com/users/chr7stos/events{/privacy}","received_events_url":"https://api.github.com/users/chr7stos/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-07-25T21:06:43Z","updated_at":"2016-07-26T19:05:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1b4fd9778b7d936af77fb4193838d2180acf5339","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/WorldBrain/cortex/pulls/47/commits","review_comments_url":"https://api.github.com/repos/WorldBrain/cortex/pulls/47/comments","review_comment_url":"https://api.github.com/repos/WorldBrain/cortex/pulls/comments{/number}","comments_url":"https://api.github.com/repos/WorldBrain/cortex/issues/47/comments","statuses_url":"https://api.github.com/repos/WorldBrain/cortex/statuses/d4168988c9940990620d3394fec950e02d9f639f","head":{"label":"chr7stos:facebook_crawler","ref":"facebook_crawler","sha":"d4168988c9940990620d3394fec950e02d9f639f","user":{"login":"chr7stos","id":3687203,"avatar_url":"https://avatars.githubusercontent.com/u/3687203?v=3","gravatar_id":"","url":"https://api.github.com/users/chr7stos","html_url":"https://github.com/chr7stos","followers_url":"https://api.github.com/users/chr7stos/followers","following_url":"https://api.github.com/users/chr7stos/following{/other_user}","gists_url":"https://api.github.com/users/chr7stos/gists{/gist_id}","starred_url":"https://api.github.com/users/chr7stos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chr7stos/subscriptions","organizations_url":"https://api.github.com/users/chr7stos/orgs","repos_url":"https://api.github.com/users/chr7stos/repos","events_url":"https://api.github.com/users/chr7stos/events{/privacy}","received_events_url":"https://api.github.com/users/chr7stos/received_events","type":"User","site_admin":false},"repo":{"id":64167530,"name":"cortex","full_name":"chr7stos/cortex","owner":{"login":"chr7stos","id":3687203,"avatar_url":"https://avatars.githubusercontent.com/u/3687203?v=3","gravatar_id":"","url":"https://api.github.com/users/chr7stos","html_url":"https://github.com/chr7stos","followers_url":"https://api.github.com/users/chr7stos/followers","following_url":"https://api.github.com/users/chr7stos/following{/other_user}","gists_url":"https://api.github.com/users/chr7stos/gists{/gist_id}","starred_url":"https://api.github.com/users/chr7stos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chr7stos/subscriptions","organizations_url":"https://api.github.com/users/chr7stos/orgs","repos_url":"https://api.github.com/users/chr7stos/repos","events_url":"https://api.github.com/users/chr7stos/events{/privacy}","received_events_url":"https://api.github.com/users/chr7stos/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/chr7stos/cortex","description":"back-end for webmarks","fork":true,"url":"https://api.github.com/repos/chr7stos/cortex","forks_url":"https://api.github.com/repos/chr7stos/cortex/forks","keys_url":"https://api.github.com/repos/chr7stos/cortex/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chr7stos/cortex/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chr7stos/cortex/teams","hooks_url":"https://api.github.com/repos/chr7stos/cortex/hooks","issue_events_url":"https://api.github.com/repos/chr7stos/cortex/issues/events{/number}","events_url":"https://api.github.com/repos/chr7stos/cortex/events","assignees_url":"https://api.github.com/repos/chr7stos/cortex/assignees{/user}","branches_url":"https://api.github.com/repos/chr7stos/cortex/branches{/branch}","tags_url":"https://api.github.com/repos/chr7stos/cortex/tags","blobs_url":"https://api.github.com/repos/chr7stos/cortex/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chr7stos/cortex/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chr7stos/cortex/git/refs{/sha}","trees_url":"https://api.github.com/repos/chr7stos/cortex/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chr7stos/cortex/statuses/{sha}","languages_url":"https://api.github.com/repos/chr7stos/cortex/languages","stargazers_url":"https://api.github.com/repos/chr7stos/cortex/stargazers","contributors_url":"https://api.github.com/repos/chr7stos/cortex/contributors","subscribers_url":"https://api.github.com/repos/chr7stos/cortex/subscribers","subscription_url":"https://api.github.com/repos/chr7stos/cortex/subscription","commits_url":"https://api.github.com/repos/chr7stos/cortex/commits{/sha}","git_commits_url":"https://api.github.com/repos/chr7stos/cortex/git/commits{/sha}","comments_url":"https://api.github.com/repos/chr7stos/cortex/comments{/number}","issue_comment_url":"https://api.github.com/repos/chr7stos/cortex/issues/comments{/number}","contents_url":"https://api.github.com/repos/chr7stos/cortex/contents/{+path}","compare_url":"https://api.github.com/repos/chr7stos/cortex/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chr7stos/cortex/merges","archive_url":"https://api.github.com/repos/chr7stos/cortex/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chr7stos/cortex/downloads","issues_url":"https://api.github.com/repos/chr7stos/cortex/issues{/number}","pulls_url":"https://api.github.com/repos/chr7stos/cortex/pulls{/number}","milestones_url":"https://api.github.com/repos/chr7stos/cortex/milestones{/number}","notifications_url":"https://api.github.com/repos/chr7stos/cortex/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chr7stos/cortex/labels{/name}","releases_url":"https://api.github.com/repos/chr7stos/cortex/releases{/id}","deployments_url":"https://api.github.com/repos/chr7stos/cortex/deployments","created_at":"2016-07-25T20:58:11Z","updated_at":"2016-07-25T20:58:12Z","pushed_at":"2016-07-25T21:05:40Z","git_url":"git://github.com/chr7stos/cortex.git","ssh_url":"git@github.com:chr7stos/cortex.git","clone_url":"https://github.com/chr7stos/cortex.git","svn_url":"https://github.com/chr7stos/cortex","homepage":"https://github.com/WorldBrain/cortex","size":78,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"WorldBrain:master","ref":"master","sha":"671097d706d9367b3dcc5a9b0206b61a3b0a5495","user":{"login":"WorldBrain","id":9511853,"avatar_url":"https://avatars.githubusercontent.com/u/9511853?v=3","gravatar_id":"","url":"https://api.github.com/users/WorldBrain","html_url":"https://github.com/WorldBrain","followers_url":"https://api.github.com/users/WorldBrain/followers","following_url":"https://api.github.com/users/WorldBrain/following{/other_user}","gists_url":"https://api.github.com/users/WorldBrain/gists{/gist_id}","starred_url":"https://api.github.com/users/WorldBrain/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WorldBrain/subscriptions","organizations_url":"https://api.github.com/users/WorldBrain/orgs","repos_url":"https://api.github.com/users/WorldBrain/repos","events_url":"https://api.github.com/users/WorldBrain/events{/privacy}","received_events_url":"https://api.github.com/users/WorldBrain/received_events","type":"Organization","site_admin":false},"repo":{"id":62507021,"name":"cortex","full_name":"WorldBrain/cortex","owner":{"login":"WorldBrain","id":9511853,"avatar_url":"https://avatars.githubusercontent.com/u/9511853?v=3","gravatar_id":"","url":"https://api.github.com/users/WorldBrain","html_url":"https://github.com/WorldBrain","followers_url":"https://api.github.com/users/WorldBrain/followers","following_url":"https://api.github.com/users/WorldBrain/following{/other_user}","gists_url":"https://api.github.com/users/WorldBrain/gists{/gist_id}","starred_url":"https://api.github.com/users/WorldBrain/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WorldBrain/subscriptions","organizations_url":"https://api.github.com/users/WorldBrain/orgs","repos_url":"https://api.github.com/users/WorldBrain/repos","events_url":"https://api.github.com/users/WorldBrain/events{/privacy}","received_events_url":"https://api.github.com/users/WorldBrain/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/WorldBrain/cortex","description":"back-end for webmarks","fork":false,"url":"https://api.github.com/repos/WorldBrain/cortex","forks_url":"https://api.github.com/repos/WorldBrain/cortex/forks","keys_url":"https://api.github.com/repos/WorldBrain/cortex/keys{/key_id}","collaborators_url":"https://api.github.com/repos/WorldBrain/cortex/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/WorldBrain/cortex/teams","hooks_url":"https://api.github.com/repos/WorldBrain/cortex/hooks","issue_events_url":"https://api.github.com/repos/WorldBrain/cortex/issues/events{/number}","events_url":"https://api.github.com/repos/WorldBrain/cortex/events","assignees_url":"https://api.github.com/repos/WorldBrain/cortex/assignees{/user}","branches_url":"https://api.github.com/repos/WorldBrain/cortex/branches{/branch}","tags_url":"https://api.github.com/repos/WorldBrain/cortex/tags","blobs_url":"https://api.github.com/repos/WorldBrain/cortex/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/WorldBrain/cortex/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/WorldBrain/cortex/git/refs{/sha}","trees_url":"https://api.github.com/repos/WorldBrain/cortex/git/trees{/sha}","statuses_url":"https://api.github.com/repos/WorldBrain/cortex/statuses/{sha}","languages_url":"https://api.github.com/repos/WorldBrain/cortex/languages","stargazers_url":"https://api.github.com/repos/WorldBrain/cortex/stargazers","contributors_url":"https://api.github.com/repos/WorldBrain/cortex/contributors","subscribers_url":"https://api.github.com/repos/WorldBrain/cortex/subscribers","subscription_url":"https://api.github.com/repos/WorldBrain/cortex/subscription","commits_url":"https://api.github.com/repos/WorldBrain/cortex/commits{/sha}","git_commits_url":"https://api.github.com/repos/WorldBrain/cortex/git/commits{/sha}","comments_url":"https://api.github.com/repos/WorldBrain/cortex/comments{/number}","issue_comment_url":"https://api.github.com/repos/WorldBrain/cortex/issues/comments{/number}","contents_url":"https://api.github.com/repos/WorldBrain/cortex/contents/{+path}","compare_url":"https://api.github.com/repos/WorldBrain/cortex/compare/{base}...{head}","merges_url":"https://api.github.com/repos/WorldBrain/cortex/merges","archive_url":"https://api.github.com/repos/WorldBrain/cortex/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/WorldBrain/cortex/downloads","issues_url":"https://api.github.com/repos/WorldBrain/cortex/issues{/number}","pulls_url":"https://api.github.com/repos/WorldBrain/cortex/pulls{/number}","milestones_url":"https://api.github.com/repos/WorldBrain/cortex/milestones{/number}","notifications_url":"https://api.github.com/repos/WorldBrain/cortex/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/WorldBrain/cortex/labels{/name}","releases_url":"https://api.github.com/repos/WorldBrain/cortex/releases{/id}","deployments_url":"https://api.github.com/repos/WorldBrain/cortex/deployments","created_at":"2016-07-03T17:12:37Z","updated_at":"2016-07-12T22:01:28Z","pushed_at":"2016-07-26T01:22:46Z","git_url":"git://github.com/WorldBrain/cortex.git","ssh_url":"git@github.com:WorldBrain/cortex.git","clone_url":"https://github.com/WorldBrain/cortex.git","svn_url":"https://github.com/WorldBrain/cortex","homepage":"https://github.com/WorldBrain/cortex","size":103,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":18,"forks":5,"open_issues":18,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/47"},"html":{"href":"https://github.com/WorldBrain/cortex/pull/47"},"issue":{"href":"https://api.github.com/repos/WorldBrain/cortex/issues/47"},"comments":{"href":"https://api.github.com/repos/WorldBrain/cortex/issues/47/comments"},"review_comments":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/47/comments"},"review_comment":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/WorldBrain/cortex/pulls/47/commits"},"statuses":{"href":"https://api.github.com/repos/WorldBrain/cortex/statuses/d4168988c9940990620d3394fec950e02d9f639f"}}}}
{"actor":{"display_login":"lwm"}}