{"action":"created","comment":{"url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/comments/54681689","id":54681689,"diff_hunk":"@@ -0,0 +1,163 @@\n+package com.test.scrapers;\n+\n+import org.jsoup.Jsoup;\n+import org.jsoup.nodes.Document;\n+import org.jsoup.nodes.Element;\n+import org.jsoup.select.Elements;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import org.openqa.selenium.WebDriver;\n+import com.machinepublishers.jbrowserdriver.Timezone;\n+import com.machinepublishers.jbrowserdriver.JBrowserDriver;\n+import com.machinepublishers.jbrowserdriver.Settings;\n+\n+import java.io.IOException;\n+import java.net.URI;\n+import java.net.URISyntaxException;\n+import java.util.HashSet;\n+import java.util.Properties;\n+import java.util.Set;\n+import java.util.regex.Matcher;\n+import java.util.regex.Pattern;\n+\n+/**\n+ * Web scraper that uses JSoup (http://jsoup.org)\n+ */\n+public class JsoupScraper implements Scraper {\n+\n+    private static final Logger logger = LoggerFactory.getLogger(JsoupScraper.class);\n+\n+    private final Set<String> visitedUrls = new HashSet<>();\n+    private final Set<String> emails = new HashSet<>();\n+    private final Pattern emailRegEx = Pattern.compile(\"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\\\.[a-zA-Z0-9-.]+\");\n+    private final boolean useWebClient;\n+\n+    public JsoupScraper(boolean useWebClient) {\n+        this.useWebClient = useWebClient;\n+    }\n+\n+    public Set<String> getEmails(String url) throws Exception {\n+        if (!url.contains(\"://\")) {\n+            url = \"http://\" + url;\n+        }\n+        URI rootUri = null;\n+        try {\n+            rootUri = new URI(url);\n+        } catch (URISyntaxException e) {\n+            System.err.println(\"Invalid URL: \" + url);\n+            System.exit(-1);\n+        }\n+\n+        try {\n+            scrapeEmailsFromPage(rootUri, url);\n+        } catch (Exception ex) {\n+            if (emails.size() == 0) {\n+                throw ex;\n+            } else {\n+                // just log a debug warning because we got some useful results back\n+                logger.warn(\"Could not finish scraping site: \" + ex);\n+            }\n+        }\n+\n+        return emails;\n+    }\n+\n+    private void scrapeEmailsFromPage(URI rootUri, String path) throws Exception {\n+        if (path.startsWith(\"javascript:\") || path.startsWith(\"#\")) {\n+            return;\n+        }\n+        if (path.startsWith(\"mailto:\")) {\n+            String email = path.substring(\"mailto\".length()+1);\n+            emails.add(email);\n+            return;\n+        }\n+\n+        String url;\n+        if (path.contains(\"://\")) {\n+            URI pathUri = new URI(path);\n+            if (!pathUri.getHost().contains(rootUri.getHost())) {\n+                // don't follow links off site\n+                return;\n+            }\n+            url = path;\n+        } else {\n+            url = rootUri.toString();\n+            if (!path.startsWith(\"/\")) {\n+                url += \"/\";\n+            }\n+            url += path;\n+        }\n+\n+        // strips off protocol header\n+        URI uri = new URI(url);\n+        String localPath = uri.getSchemeSpecificPart() + uri.getRawPath();\n+        if ((localPath.length() > 0) && visitedUrls.contains(localPath)) {\n+            // prevent circular loops\n+            return;\n+        } else {\n+            visitedUrls.add(localPath);\n+        }\n+        logger.debug(\"Scraping \" + url);\n+        System.out.print(\".\");  // progress indicator\n+\n+        Document doc = useWebClient ?\n+                Jsoup.parse(loadPageSelenium(url))\n+                : Jsoup.connect(url).get();\n+\n+        logger.debug(\"Document is \" + doc.outerHtml());\n+\n+        // extract emails\n+        Matcher matcher = emailRegEx.matcher(doc.text());\n+        while (matcher.find()) {\n+            String email = matcher.group();\n+            emails.add(email);\n+\n+            logger.info(\"Found email: \" + email);\n+            System.out.print(\"+\");  // progress indicator\n+        }\n+\n+        // now walk all the links on the page recursively\n+        Elements elements = doc.select(\"a[href]\");\n+        for (Element e : elements) {\n+            String childUrl = e.attr(\"href\");\n+            if (childUrl != null) {\n+                try {\n+                    scrapeEmailsFromPage(rootUri, childUrl);\n+                } catch (IOException | URISyntaxException ex) {\n+                    // just log exceptions\n+                    logger.warn(\"Error scraping '\" + childUrl + \"': \" + ex);\n+                }\n+            }\n+        }\n+    }\n+\n+    private String loadPageSelenium(String url) {\n+        // this can throw a lot of different exceptions\n+\n+        // disable SSL checking (set to \"compatible\" for Firefox SSL certs)\n+        Properties props = System.getProperties();\n+        props.setProperty(\"jbd.pemfile\", \"trustanything\");\n+\n+        JBrowserDriver driver = new JBrowserDriver(Settings.builder().\n+                timezone(Timezone.AMERICA_NEWYORK).build());\n+\n+        // This will block for the page load and any\n+        // associated AJAX requests\n+        driver.get(url);\n+\n+        // ignore pages w/ errors\n+        if (driver.getStatusCode() != 200) {\n+            return \"\";\n+        }\n+\n+        // Returns the page source in its current state, including\n+        // any DOM updates that occurred after page load\n+        String result = driver.getPageSource();\n+\n+        // Close the browser. Allows the Selenium thread to terminate.\n+        driver.quit();","path":"src/main/java/com/test/scrapers/JsoupScraper.java","position":159,"original_position":159,"commit_id":"25c98f1c81c2cf4bed41a3701906b92845c20b75","original_commit_id":"25c98f1c81c2cf4bed41a3701906b92845c20b75","user":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"body":"yes...thought it was safer in case it got into a bad state.  I didn't see much difference in speed...I did switch it over so it doesn't keep recreating the driver.","created_at":"2016-03-02T05:38:35Z","updated_at":"2016-03-02T05:38:35Z","html_url":"https://github.com/kenyee/EmailScraper/pull/1#discussion_r54681689","pull_request_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/comments/54681689"},"html":{"href":"https://github.com/kenyee/EmailScraper/pull/1#discussion_r54681689"},"pull_request":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1","id":60968513,"html_url":"https://github.com/kenyee/EmailScraper/pull/1","diff_url":"https://github.com/kenyee/EmailScraper/pull/1.diff","patch_url":"https://github.com/kenyee/EmailScraper/pull/1.patch","issue_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/1","number":1,"state":"open","locked":false,"title":"added jsoup/htmlunit support","user":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"body":"For AJAX sites, probably needs to use Selenium webdriver.","created_at":"2016-02-27T22:50:00Z","updated_at":"2016-03-02T05:38:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9d282ef5418b07f6b07ec7c715878291a39a1782","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1/commits","review_comments_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1/comments","review_comment_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/1/comments","statuses_url":"https://api.github.com/repos/kenyee/EmailScraper/statuses/25c98f1c81c2cf4bed41a3701906b92845c20b75","head":{"label":"kenyee:gist","ref":"gist","sha":"25c98f1c81c2cf4bed41a3701906b92845c20b75","user":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"repo":{"id":52677778,"name":"EmailScraper","full_name":"kenyee/EmailScraper","owner":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kenyee/EmailScraper","description":"Experimenting w/ web scraping tools","fork":false,"url":"https://api.github.com/repos/kenyee/EmailScraper","forks_url":"https://api.github.com/repos/kenyee/EmailScraper/forks","keys_url":"https://api.github.com/repos/kenyee/EmailScraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kenyee/EmailScraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kenyee/EmailScraper/teams","hooks_url":"https://api.github.com/repos/kenyee/EmailScraper/hooks","issue_events_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/events{/number}","events_url":"https://api.github.com/repos/kenyee/EmailScraper/events","assignees_url":"https://api.github.com/repos/kenyee/EmailScraper/assignees{/user}","branches_url":"https://api.github.com/repos/kenyee/EmailScraper/branches{/branch}","tags_url":"https://api.github.com/repos/kenyee/EmailScraper/tags","blobs_url":"https://api.github.com/repos/kenyee/EmailScraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kenyee/EmailScraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kenyee/EmailScraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/kenyee/EmailScraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kenyee/EmailScraper/statuses/{sha}","languages_url":"https://api.github.com/repos/kenyee/EmailScraper/languages","stargazers_url":"https://api.github.com/repos/kenyee/EmailScraper/stargazers","contributors_url":"https://api.github.com/repos/kenyee/EmailScraper/contributors","subscribers_url":"https://api.github.com/repos/kenyee/EmailScraper/subscribers","subscription_url":"https://api.github.com/repos/kenyee/EmailScraper/subscription","commits_url":"https://api.github.com/repos/kenyee/EmailScraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/kenyee/EmailScraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/kenyee/EmailScraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/kenyee/EmailScraper/contents/{+path}","compare_url":"https://api.github.com/repos/kenyee/EmailScraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kenyee/EmailScraper/merges","archive_url":"https://api.github.com/repos/kenyee/EmailScraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kenyee/EmailScraper/downloads","issues_url":"https://api.github.com/repos/kenyee/EmailScraper/issues{/number}","pulls_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls{/number}","milestones_url":"https://api.github.com/repos/kenyee/EmailScraper/milestones{/number}","notifications_url":"https://api.github.com/repos/kenyee/EmailScraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kenyee/EmailScraper/labels{/name}","releases_url":"https://api.github.com/repos/kenyee/EmailScraper/releases{/id}","deployments_url":"https://api.github.com/repos/kenyee/EmailScraper/deployments","created_at":"2016-02-27T17:04:25Z","updated_at":"2016-02-27T22:47:45Z","pushed_at":"2016-02-28T04:22:43Z","git_url":"git://github.com/kenyee/EmailScraper.git","ssh_url":"git@github.com:kenyee/EmailScraper.git","clone_url":"https://github.com/kenyee/EmailScraper.git","svn_url":"https://github.com/kenyee/EmailScraper","homepage":null,"size":11,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"kenyee:master","ref":"master","sha":"a208db4e71fc806237398fc7f3d51565dc6962cb","user":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"repo":{"id":52677778,"name":"EmailScraper","full_name":"kenyee/EmailScraper","owner":{"login":"kenyee","id":586670,"avatar_url":"https://avatars.githubusercontent.com/u/586670?v=3","gravatar_id":"","url":"https://api.github.com/users/kenyee","html_url":"https://github.com/kenyee","followers_url":"https://api.github.com/users/kenyee/followers","following_url":"https://api.github.com/users/kenyee/following{/other_user}","gists_url":"https://api.github.com/users/kenyee/gists{/gist_id}","starred_url":"https://api.github.com/users/kenyee/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kenyee/subscriptions","organizations_url":"https://api.github.com/users/kenyee/orgs","repos_url":"https://api.github.com/users/kenyee/repos","events_url":"https://api.github.com/users/kenyee/events{/privacy}","received_events_url":"https://api.github.com/users/kenyee/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kenyee/EmailScraper","description":"Experimenting w/ web scraping tools","fork":false,"url":"https://api.github.com/repos/kenyee/EmailScraper","forks_url":"https://api.github.com/repos/kenyee/EmailScraper/forks","keys_url":"https://api.github.com/repos/kenyee/EmailScraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kenyee/EmailScraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kenyee/EmailScraper/teams","hooks_url":"https://api.github.com/repos/kenyee/EmailScraper/hooks","issue_events_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/events{/number}","events_url":"https://api.github.com/repos/kenyee/EmailScraper/events","assignees_url":"https://api.github.com/repos/kenyee/EmailScraper/assignees{/user}","branches_url":"https://api.github.com/repos/kenyee/EmailScraper/branches{/branch}","tags_url":"https://api.github.com/repos/kenyee/EmailScraper/tags","blobs_url":"https://api.github.com/repos/kenyee/EmailScraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kenyee/EmailScraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kenyee/EmailScraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/kenyee/EmailScraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kenyee/EmailScraper/statuses/{sha}","languages_url":"https://api.github.com/repos/kenyee/EmailScraper/languages","stargazers_url":"https://api.github.com/repos/kenyee/EmailScraper/stargazers","contributors_url":"https://api.github.com/repos/kenyee/EmailScraper/contributors","subscribers_url":"https://api.github.com/repos/kenyee/EmailScraper/subscribers","subscription_url":"https://api.github.com/repos/kenyee/EmailScraper/subscription","commits_url":"https://api.github.com/repos/kenyee/EmailScraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/kenyee/EmailScraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/kenyee/EmailScraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/kenyee/EmailScraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/kenyee/EmailScraper/contents/{+path}","compare_url":"https://api.github.com/repos/kenyee/EmailScraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kenyee/EmailScraper/merges","archive_url":"https://api.github.com/repos/kenyee/EmailScraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kenyee/EmailScraper/downloads","issues_url":"https://api.github.com/repos/kenyee/EmailScraper/issues{/number}","pulls_url":"https://api.github.com/repos/kenyee/EmailScraper/pulls{/number}","milestones_url":"https://api.github.com/repos/kenyee/EmailScraper/milestones{/number}","notifications_url":"https://api.github.com/repos/kenyee/EmailScraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kenyee/EmailScraper/labels{/name}","releases_url":"https://api.github.com/repos/kenyee/EmailScraper/releases{/id}","deployments_url":"https://api.github.com/repos/kenyee/EmailScraper/deployments","created_at":"2016-02-27T17:04:25Z","updated_at":"2016-02-27T22:47:45Z","pushed_at":"2016-02-28T04:22:43Z","git_url":"git://github.com/kenyee/EmailScraper.git","ssh_url":"git@github.com:kenyee/EmailScraper.git","clone_url":"https://github.com/kenyee/EmailScraper.git","svn_url":"https://github.com/kenyee/EmailScraper","homepage":null,"size":11,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1"},"html":{"href":"https://github.com/kenyee/EmailScraper/pull/1"},"issue":{"href":"https://api.github.com/repos/kenyee/EmailScraper/issues/1"},"comments":{"href":"https://api.github.com/repos/kenyee/EmailScraper/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kenyee/EmailScraper/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/kenyee/EmailScraper/statuses/25c98f1c81c2cf4bed41a3701906b92845c20b75"}}}}