{"action":"created","comment":{"url":"https://api.github.com/repos/amelehy/email_parse/pulls/comments/85649365","pull_request_review_id":6355002,"id":85649365,"diff_hunk":"@@ -0,0 +1,82 @@\n+#!/usr/bin/env python\n+\n+import sys\n+import requests\n+from bs4 import BeautifulSoup\n+from urlparse import urlparse\n+\n+visited_urls = []\n+found_email_addresses = []\n+MAX_URLS_TO_VISIT = 50\n+\n+# initialize script\n+def init():\n+    # make sure that this script is not being imported as a module\n+    if __name__ == \"__main__\":\n+        try:\n+            root_domain = sys.argv[1]\n+            init_link_discovery('http://' + root_domain)\n+            final_emails = flatten_and_dedupe(found_email_addresses)\n+            print_results(final_emails)\n+        except IndexError:\n+            print \"Error: no argument passed for root domain\"\n+            sys.exit(1)\n+\n+# initialize link discovery to recursively determine\n+# what links we should visit from the current page\n+def init_link_discovery(url):\n+    response = requests.get(url)\n+    domain_name = urlparse(response.url).netloc\n+    raw_html = response.text\n+    soup = BeautifulSoup(raw_html, 'html.parser')\n+    links_to_visit = fetch_all_links(soup, domain_name)\n+    for link in links_to_visit:\n+        if link not in visited_urls and len(visited_urls) < MAX_URLS_TO_VISIT:\n+            print 'Visiting ' + link\n+            visited_urls.append(link)\n+            emails = init_link_discovery(link)\n+            if emails:\n+                found_email_addresses.append(emails)\n+    return parse_emails(soup) \n+\n+# fetch all relevant links in given page \n+def fetch_all_links(html, domain_name):\n+    array = []\n+    # get all urls that start with 'http' and \n+    # have the root domain name as a substring\n+    for link in html.select('a[href^=http]'):","path":"main.py","position":47,"original_position":47,"commit_id":"8aade3c63d6741121eae49464546ff748d9104ba","original_commit_id":"8aade3c63d6741121eae49464546ff748d9104ba","user":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"body":"Very good point, I've gone ahead and changed it to look for all `<a>` tags, grab the urls that have the root domain as a substring of the domain or subdomain, makes sure that the url is not a \"mailto\", and then uses `urljoin` to create a full url no matter if it is a domain, filename, or relative path.","created_at":"2016-10-29T22:12:01Z","updated_at":"2016-10-29T22:12:02Z","html_url":"https://github.com/amelehy/email_parse/pull/1#discussion_r85649365","pull_request_url":"https://api.github.com/repos/amelehy/email_parse/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/comments/85649365"},"html":{"href":"https://github.com/amelehy/email_parse/pull/1#discussion_r85649365"},"pull_request":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/amelehy/email_parse/pulls/1","id":91124210,"html_url":"https://github.com/amelehy/email_parse/pull/1","diff_url":"https://github.com/amelehy/email_parse/pull/1.diff","patch_url":"https://github.com/amelehy/email_parse/pull/1.patch","issue_url":"https://api.github.com/repos/amelehy/email_parse/issues/1","number":1,"state":"open","locked":false,"title":"Development","user":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-10-27T02:44:30Z","updated_at":"2016-10-29T22:12:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4033eb1df52448ecb805cb8910a818326b20a6e3","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/amelehy/email_parse/pulls/1/commits","review_comments_url":"https://api.github.com/repos/amelehy/email_parse/pulls/1/comments","review_comment_url":"https://api.github.com/repos/amelehy/email_parse/pulls/comments{/number}","comments_url":"https://api.github.com/repos/amelehy/email_parse/issues/1/comments","statuses_url":"https://api.github.com/repos/amelehy/email_parse/statuses/8aade3c63d6741121eae49464546ff748d9104ba","head":{"label":"amelehy:development","ref":"development","sha":"8aade3c63d6741121eae49464546ff748d9104ba","user":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"repo":{"id":71963283,"name":"email_parse","full_name":"amelehy/email_parse","owner":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/amelehy/email_parse","description":null,"fork":false,"url":"https://api.github.com/repos/amelehy/email_parse","forks_url":"https://api.github.com/repos/amelehy/email_parse/forks","keys_url":"https://api.github.com/repos/amelehy/email_parse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amelehy/email_parse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amelehy/email_parse/teams","hooks_url":"https://api.github.com/repos/amelehy/email_parse/hooks","issue_events_url":"https://api.github.com/repos/amelehy/email_parse/issues/events{/number}","events_url":"https://api.github.com/repos/amelehy/email_parse/events","assignees_url":"https://api.github.com/repos/amelehy/email_parse/assignees{/user}","branches_url":"https://api.github.com/repos/amelehy/email_parse/branches{/branch}","tags_url":"https://api.github.com/repos/amelehy/email_parse/tags","blobs_url":"https://api.github.com/repos/amelehy/email_parse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amelehy/email_parse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amelehy/email_parse/git/refs{/sha}","trees_url":"https://api.github.com/repos/amelehy/email_parse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amelehy/email_parse/statuses/{sha}","languages_url":"https://api.github.com/repos/amelehy/email_parse/languages","stargazers_url":"https://api.github.com/repos/amelehy/email_parse/stargazers","contributors_url":"https://api.github.com/repos/amelehy/email_parse/contributors","subscribers_url":"https://api.github.com/repos/amelehy/email_parse/subscribers","subscription_url":"https://api.github.com/repos/amelehy/email_parse/subscription","commits_url":"https://api.github.com/repos/amelehy/email_parse/commits{/sha}","git_commits_url":"https://api.github.com/repos/amelehy/email_parse/git/commits{/sha}","comments_url":"https://api.github.com/repos/amelehy/email_parse/comments{/number}","issue_comment_url":"https://api.github.com/repos/amelehy/email_parse/issues/comments{/number}","contents_url":"https://api.github.com/repos/amelehy/email_parse/contents/{+path}","compare_url":"https://api.github.com/repos/amelehy/email_parse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amelehy/email_parse/merges","archive_url":"https://api.github.com/repos/amelehy/email_parse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amelehy/email_parse/downloads","issues_url":"https://api.github.com/repos/amelehy/email_parse/issues{/number}","pulls_url":"https://api.github.com/repos/amelehy/email_parse/pulls{/number}","milestones_url":"https://api.github.com/repos/amelehy/email_parse/milestones{/number}","notifications_url":"https://api.github.com/repos/amelehy/email_parse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amelehy/email_parse/labels{/name}","releases_url":"https://api.github.com/repos/amelehy/email_parse/releases{/id}","deployments_url":"https://api.github.com/repos/amelehy/email_parse/deployments","created_at":"2016-10-26T03:36:02Z","updated_at":"2016-10-26T03:36:02Z","pushed_at":"2016-10-27T02:52:02Z","git_url":"git://github.com/amelehy/email_parse.git","ssh_url":"git@github.com:amelehy/email_parse.git","clone_url":"https://github.com/amelehy/email_parse.git","svn_url":"https://github.com/amelehy/email_parse","homepage":null,"size":7,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"amelehy:master","ref":"master","sha":"708a97a119c066ad1ebf0d27874495ee4b2ca055","user":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"repo":{"id":71963283,"name":"email_parse","full_name":"amelehy/email_parse","owner":{"login":"amelehy","id":3266420,"avatar_url":"https://avatars.githubusercontent.com/u/3266420?v=3","gravatar_id":"","url":"https://api.github.com/users/amelehy","html_url":"https://github.com/amelehy","followers_url":"https://api.github.com/users/amelehy/followers","following_url":"https://api.github.com/users/amelehy/following{/other_user}","gists_url":"https://api.github.com/users/amelehy/gists{/gist_id}","starred_url":"https://api.github.com/users/amelehy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amelehy/subscriptions","organizations_url":"https://api.github.com/users/amelehy/orgs","repos_url":"https://api.github.com/users/amelehy/repos","events_url":"https://api.github.com/users/amelehy/events{/privacy}","received_events_url":"https://api.github.com/users/amelehy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/amelehy/email_parse","description":null,"fork":false,"url":"https://api.github.com/repos/amelehy/email_parse","forks_url":"https://api.github.com/repos/amelehy/email_parse/forks","keys_url":"https://api.github.com/repos/amelehy/email_parse/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amelehy/email_parse/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amelehy/email_parse/teams","hooks_url":"https://api.github.com/repos/amelehy/email_parse/hooks","issue_events_url":"https://api.github.com/repos/amelehy/email_parse/issues/events{/number}","events_url":"https://api.github.com/repos/amelehy/email_parse/events","assignees_url":"https://api.github.com/repos/amelehy/email_parse/assignees{/user}","branches_url":"https://api.github.com/repos/amelehy/email_parse/branches{/branch}","tags_url":"https://api.github.com/repos/amelehy/email_parse/tags","blobs_url":"https://api.github.com/repos/amelehy/email_parse/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amelehy/email_parse/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amelehy/email_parse/git/refs{/sha}","trees_url":"https://api.github.com/repos/amelehy/email_parse/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amelehy/email_parse/statuses/{sha}","languages_url":"https://api.github.com/repos/amelehy/email_parse/languages","stargazers_url":"https://api.github.com/repos/amelehy/email_parse/stargazers","contributors_url":"https://api.github.com/repos/amelehy/email_parse/contributors","subscribers_url":"https://api.github.com/repos/amelehy/email_parse/subscribers","subscription_url":"https://api.github.com/repos/amelehy/email_parse/subscription","commits_url":"https://api.github.com/repos/amelehy/email_parse/commits{/sha}","git_commits_url":"https://api.github.com/repos/amelehy/email_parse/git/commits{/sha}","comments_url":"https://api.github.com/repos/amelehy/email_parse/comments{/number}","issue_comment_url":"https://api.github.com/repos/amelehy/email_parse/issues/comments{/number}","contents_url":"https://api.github.com/repos/amelehy/email_parse/contents/{+path}","compare_url":"https://api.github.com/repos/amelehy/email_parse/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amelehy/email_parse/merges","archive_url":"https://api.github.com/repos/amelehy/email_parse/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amelehy/email_parse/downloads","issues_url":"https://api.github.com/repos/amelehy/email_parse/issues{/number}","pulls_url":"https://api.github.com/repos/amelehy/email_parse/pulls{/number}","milestones_url":"https://api.github.com/repos/amelehy/email_parse/milestones{/number}","notifications_url":"https://api.github.com/repos/amelehy/email_parse/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amelehy/email_parse/labels{/name}","releases_url":"https://api.github.com/repos/amelehy/email_parse/releases{/id}","deployments_url":"https://api.github.com/repos/amelehy/email_parse/deployments","created_at":"2016-10-26T03:36:02Z","updated_at":"2016-10-26T03:36:02Z","pushed_at":"2016-10-27T02:52:02Z","git_url":"git://github.com/amelehy/email_parse.git","ssh_url":"git@github.com:amelehy/email_parse.git","clone_url":"https://github.com/amelehy/email_parse.git","svn_url":"https://github.com/amelehy/email_parse","homepage":null,"size":7,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/1"},"html":{"href":"https://github.com/amelehy/email_parse/pull/1"},"issue":{"href":"https://api.github.com/repos/amelehy/email_parse/issues/1"},"comments":{"href":"https://api.github.com/repos/amelehy/email_parse/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/amelehy/email_parse/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/amelehy/email_parse/statuses/8aade3c63d6741121eae49464546ff748d9104ba"}}}}
{"actor":{"display_login":"amelehy"}}