{"action":"created","comment":{"url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/comments/174244727","pull_request_review_id":103565402,"id":174244727,"diff_hunk":"@@ -134,21 +174,56 @@ class WebCrawler:\n             exit(1)\n         self.session = match.group(1)\n \n-        self.set_cookie()\n-\n-        login_resp = self.POST(LOGIN_FORM, make_formstr({\n+        login_resp = self.POST(LOGIN_FORM, LOGIN_FORM, make_formstr({\n             'username': self.username,\n             'password': self.password,\n             'csrfmiddlewaretoken': self.csrf,\n             'submit': 'Log in',\n-            'next': '/fakebook/'\n+            'next': r'%2Ffakebook%2F'\n         }))\n         print(login_resp)\n+        match = re.search('[Ss]et-[Cc]ookie:\\W*sessionid=(\\w+);', login_resp)\n+        if (not match):\n+            print('Couldn\\'t extract SessionID token, aborting')\n+            print(login_resp)\n+            exit(1)\n+        self.session = match.group(1)\n+\n+        return self.traverse_to(get_location(login_resp))\n+\n+def get_linkset(origin, page):\n+    matches = re.findall('href=[\\'\"]([/\\w]+)', page)\n+    for match in matches:\n+        print(match)\n+    return set()\n \n+def hunt_key(page):\n+    return None\n \n def crawl(crawler):\n     \"\"\"Runs the web crawler\"\"\"\n-    crawler.login()\n+    visited = set()\n+    unvisited = set()\n+    secret_keys = []\n+\n+    login_page = crawler.login()\n+    print(login_page)\n+    unvisited = get_linkset(ROOT, login_page)\n+\n+    while len(unvisited) is not 0:\n+        url = unvisited.pop()\n+        visited.add(url)\n+\n+        page = crawler.traverse_to(url)\n+        if page is None:\n+            continue\n+        secret_key = hunt_key(page)\n+        if secret_key:\n+            secret_keys.append(secret_key)\n+\n+        unvisited |= (get_linkset(ROOT, page) - visited)","path":"project4/3700crawler","position":176,"original_position":176,"commit_id":"5387d554bd496a866844aff74fdf828bdec32cab","original_commit_id":"5387d554bd496a866844aff74fdf828bdec32cab","user":{"login":"kimberlypn","id":19919068,"avatar_url":"https://avatars0.githubusercontent.com/u/19919068?v=4","gravatar_id":"","url":"https://api.github.com/users/kimberlypn","html_url":"https://github.com/kimberlypn","followers_url":"https://api.github.com/users/kimberlypn/followers","following_url":"https://api.github.com/users/kimberlypn/following{/other_user}","gists_url":"https://api.github.com/users/kimberlypn/gists{/gist_id}","starred_url":"https://api.github.com/users/kimberlypn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kimberlypn/subscriptions","organizations_url":"https://api.github.com/users/kimberlypn/orgs","repos_url":"https://api.github.com/users/kimberlypn/repos","events_url":"https://api.github.com/users/kimberlypn/events{/privacy}","received_events_url":"https://api.github.com/users/kimberlypn/received_events","type":"User","site_admin":false},"body":"Ah, okay. I was just unsure about the `|=` (haven't started using Python until this class).","created_at":"2018-03-13T18:46:26Z","updated_at":"2018-03-13T18:46:26Z","html_url":"https://github.com/kimberlypn/CS3700/pull/4#discussion_r174244727","pull_request_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/comments/174244727"},"html":{"href":"https://github.com/kimberlypn/CS3700/pull/4#discussion_r174244727"},"pull_request":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4"}},"in_reply_to_id":174002944},"pull_request":{"url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4","id":174532827,"html_url":"https://github.com/kimberlypn/CS3700/pull/4","diff_url":"https://github.com/kimberlypn/CS3700/pull/4.diff","patch_url":"https://github.com/kimberlypn/CS3700/pull/4.patch","issue_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/4","number":4,"state":"open","locked":false,"title":"Login works properly","user":{"login":"sweeneytr","id":8851347,"avatar_url":"https://avatars0.githubusercontent.com/u/8851347?v=4","gravatar_id":"","url":"https://api.github.com/users/sweeneytr","html_url":"https://github.com/sweeneytr","followers_url":"https://api.github.com/users/sweeneytr/followers","following_url":"https://api.github.com/users/sweeneytr/following{/other_user}","gists_url":"https://api.github.com/users/sweeneytr/gists{/gist_id}","starred_url":"https://api.github.com/users/sweeneytr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sweeneytr/subscriptions","organizations_url":"https://api.github.com/users/sweeneytr/orgs","repos_url":"https://api.github.com/users/sweeneytr/repos","events_url":"https://api.github.com/users/sweeneytr/events{/privacy}","received_events_url":"https://api.github.com/users/sweeneytr/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-03-13T00:55:58Z","updated_at":"2018-03-13T18:46:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f758b62f72b0ebdf6192b5e4d563a7901913718f","assignee":{"login":"sweeneytr","id":8851347,"avatar_url":"https://avatars0.githubusercontent.com/u/8851347?v=4","gravatar_id":"","url":"https://api.github.com/users/sweeneytr","html_url":"https://github.com/sweeneytr","followers_url":"https://api.github.com/users/sweeneytr/followers","following_url":"https://api.github.com/users/sweeneytr/following{/other_user}","gists_url":"https://api.github.com/users/sweeneytr/gists{/gist_id}","starred_url":"https://api.github.com/users/sweeneytr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sweeneytr/subscriptions","organizations_url":"https://api.github.com/users/sweeneytr/orgs","repos_url":"https://api.github.com/users/sweeneytr/repos","events_url":"https://api.github.com/users/sweeneytr/events{/privacy}","received_events_url":"https://api.github.com/users/sweeneytr/received_events","type":"User","site_admin":false},"assignees":[{"login":"sweeneytr","id":8851347,"avatar_url":"https://avatars0.githubusercontent.com/u/8851347?v=4","gravatar_id":"","url":"https://api.github.com/users/sweeneytr","html_url":"https://github.com/sweeneytr","followers_url":"https://api.github.com/users/sweeneytr/followers","following_url":"https://api.github.com/users/sweeneytr/following{/other_user}","gists_url":"https://api.github.com/users/sweeneytr/gists{/gist_id}","starred_url":"https://api.github.com/users/sweeneytr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sweeneytr/subscriptions","organizations_url":"https://api.github.com/users/sweeneytr/orgs","repos_url":"https://api.github.com/users/sweeneytr/repos","events_url":"https://api.github.com/users/sweeneytr/events{/privacy}","received_events_url":"https://api.github.com/users/sweeneytr/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4/commits","review_comments_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4/comments","review_comment_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/4/comments","statuses_url":"https://api.github.com/repos/kimberlypn/CS3700/statuses/5387d554bd496a866844aff74fdf828bdec32cab","head":{"label":"kimberlypn:tristan-refhunter","ref":"tristan-refhunter","sha":"5387d554bd496a866844aff74fdf828bdec32cab","user":{"login":"kimberlypn","id":19919068,"avatar_url":"https://avatars0.githubusercontent.com/u/19919068?v=4","gravatar_id":"","url":"https://api.github.com/users/kimberlypn","html_url":"https://github.com/kimberlypn","followers_url":"https://api.github.com/users/kimberlypn/followers","following_url":"https://api.github.com/users/kimberlypn/following{/other_user}","gists_url":"https://api.github.com/users/kimberlypn/gists{/gist_id}","starred_url":"https://api.github.com/users/kimberlypn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kimberlypn/subscriptions","organizations_url":"https://api.github.com/users/kimberlypn/orgs","repos_url":"https://api.github.com/users/kimberlypn/repos","events_url":"https://api.github.com/users/kimberlypn/events{/privacy}","received_events_url":"https://api.github.com/users/kimberlypn/received_events","type":"User","site_admin":false},"repo":{"id":124795738,"name":"CS3700","full_name":"kimberlypn/CS3700","owner":{"login":"kimberlypn","id":19919068,"avatar_url":"https://avatars0.githubusercontent.com/u/19919068?v=4","gravatar_id":"","url":"https://api.github.com/users/kimberlypn","html_url":"https://github.com/kimberlypn","followers_url":"https://api.github.com/users/kimberlypn/followers","following_url":"https://api.github.com/users/kimberlypn/following{/other_user}","gists_url":"https://api.github.com/users/kimberlypn/gists{/gist_id}","starred_url":"https://api.github.com/users/kimberlypn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kimberlypn/subscriptions","organizations_url":"https://api.github.com/users/kimberlypn/orgs","repos_url":"https://api.github.com/users/kimberlypn/repos","events_url":"https://api.github.com/users/kimberlypn/events{/privacy}","received_events_url":"https://api.github.com/users/kimberlypn/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kimberlypn/CS3700","description":"CS3700 Networks and Distributed Systems (Spring 2018)","fork":false,"url":"https://api.github.com/repos/kimberlypn/CS3700","forks_url":"https://api.github.com/repos/kimberlypn/CS3700/forks","keys_url":"https://api.github.com/repos/kimberlypn/CS3700/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kimberlypn/CS3700/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kimberlypn/CS3700/teams","hooks_url":"https://api.github.com/repos/kimberlypn/CS3700/hooks","issue_events_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/events{/number}","events_url":"https://api.github.com/repos/kimberlypn/CS3700/events","assignees_url":"https://api.github.com/repos/kimberlypn/CS3700/assignees{/user}","branches_url":"https://api.github.com/repos/kimberlypn/CS3700/branches{/branch}","tags_url":"https://api.github.com/repos/kimberlypn/CS3700/tags","blobs_url":"https://api.github.com/repos/kimberlypn/CS3700/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kimberlypn/CS3700/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kimberlypn/CS3700/git/refs{/sha}","trees_url":"https://api.github.com/repos/kimberlypn/CS3700/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kimberlypn/CS3700/statuses/{sha}","languages_url":"https://api.github.com/repos/kimberlypn/CS3700/languages","stargazers_url":"https://api.github.com/repos/kimberlypn/CS3700/stargazers","contributors_url":"https://api.github.com/repos/kimberlypn/CS3700/contributors","subscribers_url":"https://api.github.com/repos/kimberlypn/CS3700/subscribers","subscription_url":"https://api.github.com/repos/kimberlypn/CS3700/subscription","commits_url":"https://api.github.com/repos/kimberlypn/CS3700/commits{/sha}","git_commits_url":"https://api.github.com/repos/kimberlypn/CS3700/git/commits{/sha}","comments_url":"https://api.github.com/repos/kimberlypn/CS3700/comments{/number}","issue_comment_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/comments{/number}","contents_url":"https://api.github.com/repos/kimberlypn/CS3700/contents/{+path}","compare_url":"https://api.github.com/repos/kimberlypn/CS3700/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kimberlypn/CS3700/merges","archive_url":"https://api.github.com/repos/kimberlypn/CS3700/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kimberlypn/CS3700/downloads","issues_url":"https://api.github.com/repos/kimberlypn/CS3700/issues{/number}","pulls_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls{/number}","milestones_url":"https://api.github.com/repos/kimberlypn/CS3700/milestones{/number}","notifications_url":"https://api.github.com/repos/kimberlypn/CS3700/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kimberlypn/CS3700/labels{/name}","releases_url":"https://api.github.com/repos/kimberlypn/CS3700/releases{/id}","deployments_url":"https://api.github.com/repos/kimberlypn/CS3700/deployments","created_at":"2018-03-11T20:18:56Z","updated_at":"2018-03-12T17:45:44Z","pushed_at":"2018-03-13T04:44:38Z","git_url":"git://github.com/kimberlypn/CS3700.git","ssh_url":"git@github.com:kimberlypn/CS3700.git","clone_url":"https://github.com/kimberlypn/CS3700.git","svn_url":"https://github.com/kimberlypn/CS3700","homepage":null,"size":17,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"kimberlypn:master","ref":"master","sha":"746a3b45e2d29067803bea8525b1d6100dd039b6","user":{"login":"kimberlypn","id":19919068,"avatar_url":"https://avatars0.githubusercontent.com/u/19919068?v=4","gravatar_id":"","url":"https://api.github.com/users/kimberlypn","html_url":"https://github.com/kimberlypn","followers_url":"https://api.github.com/users/kimberlypn/followers","following_url":"https://api.github.com/users/kimberlypn/following{/other_user}","gists_url":"https://api.github.com/users/kimberlypn/gists{/gist_id}","starred_url":"https://api.github.com/users/kimberlypn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kimberlypn/subscriptions","organizations_url":"https://api.github.com/users/kimberlypn/orgs","repos_url":"https://api.github.com/users/kimberlypn/repos","events_url":"https://api.github.com/users/kimberlypn/events{/privacy}","received_events_url":"https://api.github.com/users/kimberlypn/received_events","type":"User","site_admin":false},"repo":{"id":124795738,"name":"CS3700","full_name":"kimberlypn/CS3700","owner":{"login":"kimberlypn","id":19919068,"avatar_url":"https://avatars0.githubusercontent.com/u/19919068?v=4","gravatar_id":"","url":"https://api.github.com/users/kimberlypn","html_url":"https://github.com/kimberlypn","followers_url":"https://api.github.com/users/kimberlypn/followers","following_url":"https://api.github.com/users/kimberlypn/following{/other_user}","gists_url":"https://api.github.com/users/kimberlypn/gists{/gist_id}","starred_url":"https://api.github.com/users/kimberlypn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kimberlypn/subscriptions","organizations_url":"https://api.github.com/users/kimberlypn/orgs","repos_url":"https://api.github.com/users/kimberlypn/repos","events_url":"https://api.github.com/users/kimberlypn/events{/privacy}","received_events_url":"https://api.github.com/users/kimberlypn/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kimberlypn/CS3700","description":"CS3700 Networks and Distributed Systems (Spring 2018)","fork":false,"url":"https://api.github.com/repos/kimberlypn/CS3700","forks_url":"https://api.github.com/repos/kimberlypn/CS3700/forks","keys_url":"https://api.github.com/repos/kimberlypn/CS3700/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kimberlypn/CS3700/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kimberlypn/CS3700/teams","hooks_url":"https://api.github.com/repos/kimberlypn/CS3700/hooks","issue_events_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/events{/number}","events_url":"https://api.github.com/repos/kimberlypn/CS3700/events","assignees_url":"https://api.github.com/repos/kimberlypn/CS3700/assignees{/user}","branches_url":"https://api.github.com/repos/kimberlypn/CS3700/branches{/branch}","tags_url":"https://api.github.com/repos/kimberlypn/CS3700/tags","blobs_url":"https://api.github.com/repos/kimberlypn/CS3700/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kimberlypn/CS3700/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kimberlypn/CS3700/git/refs{/sha}","trees_url":"https://api.github.com/repos/kimberlypn/CS3700/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kimberlypn/CS3700/statuses/{sha}","languages_url":"https://api.github.com/repos/kimberlypn/CS3700/languages","stargazers_url":"https://api.github.com/repos/kimberlypn/CS3700/stargazers","contributors_url":"https://api.github.com/repos/kimberlypn/CS3700/contributors","subscribers_url":"https://api.github.com/repos/kimberlypn/CS3700/subscribers","subscription_url":"https://api.github.com/repos/kimberlypn/CS3700/subscription","commits_url":"https://api.github.com/repos/kimberlypn/CS3700/commits{/sha}","git_commits_url":"https://api.github.com/repos/kimberlypn/CS3700/git/commits{/sha}","comments_url":"https://api.github.com/repos/kimberlypn/CS3700/comments{/number}","issue_comment_url":"https://api.github.com/repos/kimberlypn/CS3700/issues/comments{/number}","contents_url":"https://api.github.com/repos/kimberlypn/CS3700/contents/{+path}","compare_url":"https://api.github.com/repos/kimberlypn/CS3700/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kimberlypn/CS3700/merges","archive_url":"https://api.github.com/repos/kimberlypn/CS3700/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kimberlypn/CS3700/downloads","issues_url":"https://api.github.com/repos/kimberlypn/CS3700/issues{/number}","pulls_url":"https://api.github.com/repos/kimberlypn/CS3700/pulls{/number}","milestones_url":"https://api.github.com/repos/kimberlypn/CS3700/milestones{/number}","notifications_url":"https://api.github.com/repos/kimberlypn/CS3700/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kimberlypn/CS3700/labels{/name}","releases_url":"https://api.github.com/repos/kimberlypn/CS3700/releases{/id}","deployments_url":"https://api.github.com/repos/kimberlypn/CS3700/deployments","created_at":"2018-03-11T20:18:56Z","updated_at":"2018-03-12T17:45:44Z","pushed_at":"2018-03-13T04:44:38Z","git_url":"git://github.com/kimberlypn/CS3700.git","ssh_url":"git@github.com:kimberlypn/CS3700.git","clone_url":"https://github.com/kimberlypn/CS3700.git","svn_url":"https://github.com/kimberlypn/CS3700","homepage":null,"size":17,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4"},"html":{"href":"https://github.com/kimberlypn/CS3700/pull/4"},"issue":{"href":"https://api.github.com/repos/kimberlypn/CS3700/issues/4"},"comments":{"href":"https://api.github.com/repos/kimberlypn/CS3700/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kimberlypn/CS3700/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/kimberlypn/CS3700/statuses/5387d554bd496a866844aff74fdf828bdec32cab"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"kimberlypn"}}