{"action":"created","comment":{"url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/comments/169171049","pull_request_review_id":97650784,"id":169171049,"diff_hunk":"@@ -0,0 +1,196 @@\n+\"\"\"Various functions for extracting links from stories and for storing them in topics.\"\"\"\n+\n+from bs4 import BeautifulSoup\n+import re\n+import typing\n+\n+from mediawords.db import DatabaseHandler\n+import mediawords.dbi.downloads\n+from mediawords.util.log import create_logger\n+\n+log = create_logger(__name__)\n+\n+\n+_IGNORE_LINK_PATTERN = (\n+    '(www.addtoany.com)|(novostimira.com)|(ads\\.pheedo)|(www.dailykos.com\\/user)|'\n+    '(livejournal.com\\/(tag|profile))|(sfbayview.com\\/tag)|(absoluteastronomy.com)|'\n+    '(\\/share.*http)|(digg.com\\/submit)|(facebook.com.*mediacontentsharebutton)|'\n+    '(feeds.wordpress.com\\/.*\\/go)|(sharetodiaspora.github.io\\/)|(iconosquare.com)|'\n+    '(unz.com)|(answers.com)|(downwithtyranny.com\\/search)|(scoop\\.?it)|(sco\\.lt)|'\n+    '(pronk.*\\.wordpress\\.com\\/(tag|category))|(wn\\.com)|(pinterest\\.com\\/pin\\/create)|(feedblitz\\.com)|(atomz.com)')\n+\n+\n+def get_links_from_html(html: str) -> typing.List[str]:\n+    \"\"\"Return a list of all links that appear in the html.\n+\n+    Only return absolute urls, because we would rather get fewer internal media source links.  Also include embedded\n+    youtube video urls.\n+\n+    Arguments:\n+    html - html to parse\n+\n+    Returns:\n+    list of string urls\n+\n+    \"\"\"\n+    soup = BeautifulSoup(html, 'lxml')\n+\n+    links = []\n+\n+    # get everything with an href= element rather than just <a /> links\n+    for tag in soup.find_all(href=True):\n+        url = tag['href']\n+\n+        if re.search(_IGNORE_LINK_PATTERN, url, flags=re.I) is not None:\n+            continue\n+\n+        if not mediawords.util.url.is_http_url(url):\n+            continue\n+\n+        url = re.sub(r'www[a-z0-9]+.nytimes', 'www.nytimes', url, flags=re.I)\n+\n+        links.append(url)\n+\n+    return links\n+\n+\n+def get_youtube_embed_links(db: DatabaseHandler, story: dict) -> typing.List[str]:\n+    \"\"\"Parse youtube embedded video urls out of the full html of the story.\n+\n+    This function looks for youtube embed links anywhere in the html of the story content, rather than just in the\n+    extracted html.  It aims to return a superset of all youtube embed links by returning every iframe src= attribute\n+    that includes the string 'youtube'.\n+\n+    Arguments:\n+    db - db handle\n+    story - story dict from db\n+\n+    Returns:\n+    list of string urls\n+\n+    \"\"\"\n+    download = db.query(\n+        \"select * from downloads where stories_id = %(a)s order by stories_id limit 1\",\n+        {'a': story['stories_id']}).hash()\n+\n+    html = mediawords.dbi.downloads.fetch_content(db, download)\n+\n+    soup = BeautifulSoup(html, 'lxml')\n+\n+    links = []\n+    for tag in soup.find_all('iframe', src=True):\n+        url = tag['src']\n+\n+        if 'youtube' not in url:\n+            continue\n+\n+        if not url.lower().startswith('http'):\n+            url = 'http:' + url\n+\n+        url = url.strip()\n+\n+        url = re.sub('youtube-embed', 'youtube', url)\n+\n+        links.append(url)\n+\n+    return links\n+\n+\n+def get_extracted_html(db: DatabaseHandler, story: dict) -> str:\n+    \"\"\"Get the extracted html for the story.","path":"mediacloud/mediawords/tm/extract_story_links.py","position":100,"original_position":100,"commit_id":"6338139437a0e04e75d9432f27137fe200bf312a","original_commit_id":"6338139437a0e04e75d9432f27137fe200bf312a","user":{"login":"pypt","id":181949,"avatar_url":"https://avatars3.githubusercontent.com/u/181949?v=4","gravatar_id":"","url":"https://api.github.com/users/pypt","html_url":"https://github.com/pypt","followers_url":"https://api.github.com/users/pypt/followers","following_url":"https://api.github.com/users/pypt/following{/other_user}","gists_url":"https://api.github.com/users/pypt/gists{/gist_id}","starred_url":"https://api.github.com/users/pypt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pypt/subscriptions","organizations_url":"https://api.github.com/users/pypt/orgs","repos_url":"https://api.github.com/users/pypt/repos","events_url":"https://api.github.com/users/pypt/events{/privacy}","received_events_url":"https://api.github.com/users/pypt/received_events","type":"User","site_admin":false},"body":"`story = decode_object_from_bytes_if_needed(story)`","created_at":"2018-02-19T21:01:23Z","updated_at":"2018-02-19T21:21:58Z","html_url":"https://github.com/berkmancenter/mediacloud/pull/298#discussion_r169171049","pull_request_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/comments/169171049"},"html":{"href":"https://github.com/berkmancenter/mediacloud/pull/298#discussion_r169171049"},"pull_request":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298"}}},"pull_request":{"url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298","id":169012619,"html_url":"https://github.com/berkmancenter/mediacloud/pull/298","diff_url":"https://github.com/berkmancenter/mediacloud/pull/298.diff","patch_url":"https://github.com/berkmancenter/mediacloud/pull/298.patch","issue_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/298","number":298,"state":"closed","locked":false,"title":"Distributed topic spider - extract links","user":{"login":"hroberts","id":1830735,"avatar_url":"https://avatars2.githubusercontent.com/u/1830735?v=4","gravatar_id":"","url":"https://api.github.com/users/hroberts","html_url":"https://github.com/hroberts","followers_url":"https://api.github.com/users/hroberts/followers","following_url":"https://api.github.com/users/hroberts/following{/other_user}","gists_url":"https://api.github.com/users/hroberts/gists{/gist_id}","starred_url":"https://api.github.com/users/hroberts/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hroberts/subscriptions","organizations_url":"https://api.github.com/users/hroberts/orgs","repos_url":"https://api.github.com/users/hroberts/repos","events_url":"https://api.github.com/users/hroberts/events{/privacy}","received_events_url":"https://api.github.com/users/hroberts/received_events","type":"User","site_admin":false},"body":"This pull request moves the code that extracts links from stories during the topic spidering process from the perl Topic::Mine module to the python jobs based mediawords.tm.extract_story_links.\r\n\r\nThis is the first step toward moving all of the topic spidering code from the existing, mostly single process perl module to jobs based python module.  As an intermediate step, the new python jobs are called from the exiting perl module in chunks of 1000, which just waits around for the python jobs to finish, polling the topic_links table to know when they are done.  It should only take an easy edit of the extract_story_links_job.py package to make this new code work with the fully distributed topic spider.\r\n\r\nIn addition to that top level deliverable, there are several other changes in dependencies of the new python code:\r\n\r\n* Util::HTML is now entirely in python, including the needed html_strip() function, which is now BeautifulSoup based (the old perl regex was too fancy for the python regex engine to handle, but the BS module seems to perform fine according to casual benchmarking).\r\n\r\n* DBI::Downloads was entirely translated to python so that we can call store_content, fetch_content, and the various extract functions in python.  I have not plugged the perl side into those python implementations yet because I would like to run the python DBI::Downloads code just in the link extractor code for a while before running it for the whole crawler engine.\r\n\r\n* I changed the test case class in mediawords.test.test_database to create a template database from mediawords.sql only once during each the life of each process and to use the postgres 'create mediacloud_test template mediacloud_test_template' functionality to more quickly recreate the test database for each unit test.  On my laptop, this reduced the db setup time for each unit test from about 3 seconds to about 0.5 seconds, and the run time for all python tests from about 140 seconds to about 100 seconds.  This change is included in this pull request because this PR includes lots of individual unit tests that rely on having a fresh database.\r\n\r\n* There was a get_boingboing_links() function that was no longer working with current boingboing, so I just removed it.\r\n\r\nAlso:\r\n\r\n* This PR also has a first version of a ThrottledUserAgent that will allow per media throttling of the planned topic_fetch_link() job.  That code is not used anywhere yet.\r\n\r\n* There are also some miscellaneous small fixes to make mypy happy with existing code in files that I had to touch for this PR.  Mostly, mypy was consistently unhappy when assigning decode_obj_from_bytes_if_needed()  to a str, so I added a decode_str_from_bytes_if_needed() that made it easier to make mypy happy.","created_at":"2018-02-14T03:13:24Z","updated_at":"2018-02-19T21:21:58Z","closed_at":"2018-02-14T13:54:29Z","merged_at":null,"merge_commit_sha":"d8ffc81d9fffeb37924248395e9c185e9920877f","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298/commits","review_comments_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298/comments","review_comment_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/comments{/number}","comments_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/298/comments","statuses_url":"https://api.github.com/repos/berkmancenter/mediacloud/statuses/6338139437a0e04e75d9432f27137fe200bf312a","head":{"label":"berkmancenter:distributed_topic_spider_extract_links","ref":"distributed_topic_spider_extract_links","sha":"6338139437a0e04e75d9432f27137fe200bf312a","user":{"login":"berkmancenter","id":131221,"avatar_url":"https://avatars0.githubusercontent.com/u/131221?v=4","gravatar_id":"","url":"https://api.github.com/users/berkmancenter","html_url":"https://github.com/berkmancenter","followers_url":"https://api.github.com/users/berkmancenter/followers","following_url":"https://api.github.com/users/berkmancenter/following{/other_user}","gists_url":"https://api.github.com/users/berkmancenter/gists{/gist_id}","starred_url":"https://api.github.com/users/berkmancenter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berkmancenter/subscriptions","organizations_url":"https://api.github.com/users/berkmancenter/orgs","repos_url":"https://api.github.com/users/berkmancenter/repos","events_url":"https://api.github.com/users/berkmancenter/events{/privacy}","received_events_url":"https://api.github.com/users/berkmancenter/received_events","type":"Organization","site_admin":false},"repo":{"id":10965582,"name":"mediacloud","full_name":"berkmancenter/mediacloud","owner":{"login":"berkmancenter","id":131221,"avatar_url":"https://avatars0.githubusercontent.com/u/131221?v=4","gravatar_id":"","url":"https://api.github.com/users/berkmancenter","html_url":"https://github.com/berkmancenter","followers_url":"https://api.github.com/users/berkmancenter/followers","following_url":"https://api.github.com/users/berkmancenter/following{/other_user}","gists_url":"https://api.github.com/users/berkmancenter/gists{/gist_id}","starred_url":"https://api.github.com/users/berkmancenter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berkmancenter/subscriptions","organizations_url":"https://api.github.com/users/berkmancenter/orgs","repos_url":"https://api.github.com/users/berkmancenter/repos","events_url":"https://api.github.com/users/berkmancenter/events{/privacy}","received_events_url":"https://api.github.com/users/berkmancenter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/berkmancenter/mediacloud","description":"Media Cloud is an open source, open data platform that allows researchers to answer quantitative questions about the content of online media.","fork":false,"url":"https://api.github.com/repos/berkmancenter/mediacloud","forks_url":"https://api.github.com/repos/berkmancenter/mediacloud/forks","keys_url":"https://api.github.com/repos/berkmancenter/mediacloud/keys{/key_id}","collaborators_url":"https://api.github.com/repos/berkmancenter/mediacloud/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/berkmancenter/mediacloud/teams","hooks_url":"https://api.github.com/repos/berkmancenter/mediacloud/hooks","issue_events_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/events{/number}","events_url":"https://api.github.com/repos/berkmancenter/mediacloud/events","assignees_url":"https://api.github.com/repos/berkmancenter/mediacloud/assignees{/user}","branches_url":"https://api.github.com/repos/berkmancenter/mediacloud/branches{/branch}","tags_url":"https://api.github.com/repos/berkmancenter/mediacloud/tags","blobs_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/refs{/sha}","trees_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berkmancenter/mediacloud/statuses/{sha}","languages_url":"https://api.github.com/repos/berkmancenter/mediacloud/languages","stargazers_url":"https://api.github.com/repos/berkmancenter/mediacloud/stargazers","contributors_url":"https://api.github.com/repos/berkmancenter/mediacloud/contributors","subscribers_url":"https://api.github.com/repos/berkmancenter/mediacloud/subscribers","subscription_url":"https://api.github.com/repos/berkmancenter/mediacloud/subscription","commits_url":"https://api.github.com/repos/berkmancenter/mediacloud/commits{/sha}","git_commits_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/commits{/sha}","comments_url":"https://api.github.com/repos/berkmancenter/mediacloud/comments{/number}","issue_comment_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/comments{/number}","contents_url":"https://api.github.com/repos/berkmancenter/mediacloud/contents/{+path}","compare_url":"https://api.github.com/repos/berkmancenter/mediacloud/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berkmancenter/mediacloud/merges","archive_url":"https://api.github.com/repos/berkmancenter/mediacloud/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berkmancenter/mediacloud/downloads","issues_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues{/number}","pulls_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls{/number}","milestones_url":"https://api.github.com/repos/berkmancenter/mediacloud/milestones{/number}","notifications_url":"https://api.github.com/repos/berkmancenter/mediacloud/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berkmancenter/mediacloud/labels{/name}","releases_url":"https://api.github.com/repos/berkmancenter/mediacloud/releases{/id}","deployments_url":"https://api.github.com/repos/berkmancenter/mediacloud/deployments","created_at":"2013-06-26T10:59:56Z","updated_at":"2018-02-17T22:48:58Z","pushed_at":"2018-02-19T20:05:03Z","git_url":"git://github.com/berkmancenter/mediacloud.git","ssh_url":"git@github.com:berkmancenter/mediacloud.git","clone_url":"https://github.com/berkmancenter/mediacloud.git","svn_url":"https://github.com/berkmancenter/mediacloud","homepage":"http://www.mediacloud.org","size":250222,"stargazers_count":87,"watchers_count":87,"language":"HTML","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":34,"mirror_url":null,"archived":false,"open_issues_count":54,"license":{"key":"agpl-3.0","name":"GNU Affero General Public License v3.0","spdx_id":"AGPL-3.0","url":"https://api.github.com/licenses/agpl-3.0"},"forks":34,"open_issues":54,"watchers":87,"default_branch":"master"}},"base":{"label":"berkmancenter:master","ref":"master","sha":"afbf5f93a43f8bae861d5be6e44980af68b3b4e2","user":{"login":"berkmancenter","id":131221,"avatar_url":"https://avatars0.githubusercontent.com/u/131221?v=4","gravatar_id":"","url":"https://api.github.com/users/berkmancenter","html_url":"https://github.com/berkmancenter","followers_url":"https://api.github.com/users/berkmancenter/followers","following_url":"https://api.github.com/users/berkmancenter/following{/other_user}","gists_url":"https://api.github.com/users/berkmancenter/gists{/gist_id}","starred_url":"https://api.github.com/users/berkmancenter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berkmancenter/subscriptions","organizations_url":"https://api.github.com/users/berkmancenter/orgs","repos_url":"https://api.github.com/users/berkmancenter/repos","events_url":"https://api.github.com/users/berkmancenter/events{/privacy}","received_events_url":"https://api.github.com/users/berkmancenter/received_events","type":"Organization","site_admin":false},"repo":{"id":10965582,"name":"mediacloud","full_name":"berkmancenter/mediacloud","owner":{"login":"berkmancenter","id":131221,"avatar_url":"https://avatars0.githubusercontent.com/u/131221?v=4","gravatar_id":"","url":"https://api.github.com/users/berkmancenter","html_url":"https://github.com/berkmancenter","followers_url":"https://api.github.com/users/berkmancenter/followers","following_url":"https://api.github.com/users/berkmancenter/following{/other_user}","gists_url":"https://api.github.com/users/berkmancenter/gists{/gist_id}","starred_url":"https://api.github.com/users/berkmancenter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/berkmancenter/subscriptions","organizations_url":"https://api.github.com/users/berkmancenter/orgs","repos_url":"https://api.github.com/users/berkmancenter/repos","events_url":"https://api.github.com/users/berkmancenter/events{/privacy}","received_events_url":"https://api.github.com/users/berkmancenter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/berkmancenter/mediacloud","description":"Media Cloud is an open source, open data platform that allows researchers to answer quantitative questions about the content of online media.","fork":false,"url":"https://api.github.com/repos/berkmancenter/mediacloud","forks_url":"https://api.github.com/repos/berkmancenter/mediacloud/forks","keys_url":"https://api.github.com/repos/berkmancenter/mediacloud/keys{/key_id}","collaborators_url":"https://api.github.com/repos/berkmancenter/mediacloud/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/berkmancenter/mediacloud/teams","hooks_url":"https://api.github.com/repos/berkmancenter/mediacloud/hooks","issue_events_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/events{/number}","events_url":"https://api.github.com/repos/berkmancenter/mediacloud/events","assignees_url":"https://api.github.com/repos/berkmancenter/mediacloud/assignees{/user}","branches_url":"https://api.github.com/repos/berkmancenter/mediacloud/branches{/branch}","tags_url":"https://api.github.com/repos/berkmancenter/mediacloud/tags","blobs_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/refs{/sha}","trees_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/trees{/sha}","statuses_url":"https://api.github.com/repos/berkmancenter/mediacloud/statuses/{sha}","languages_url":"https://api.github.com/repos/berkmancenter/mediacloud/languages","stargazers_url":"https://api.github.com/repos/berkmancenter/mediacloud/stargazers","contributors_url":"https://api.github.com/repos/berkmancenter/mediacloud/contributors","subscribers_url":"https://api.github.com/repos/berkmancenter/mediacloud/subscribers","subscription_url":"https://api.github.com/repos/berkmancenter/mediacloud/subscription","commits_url":"https://api.github.com/repos/berkmancenter/mediacloud/commits{/sha}","git_commits_url":"https://api.github.com/repos/berkmancenter/mediacloud/git/commits{/sha}","comments_url":"https://api.github.com/repos/berkmancenter/mediacloud/comments{/number}","issue_comment_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues/comments{/number}","contents_url":"https://api.github.com/repos/berkmancenter/mediacloud/contents/{+path}","compare_url":"https://api.github.com/repos/berkmancenter/mediacloud/compare/{base}...{head}","merges_url":"https://api.github.com/repos/berkmancenter/mediacloud/merges","archive_url":"https://api.github.com/repos/berkmancenter/mediacloud/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/berkmancenter/mediacloud/downloads","issues_url":"https://api.github.com/repos/berkmancenter/mediacloud/issues{/number}","pulls_url":"https://api.github.com/repos/berkmancenter/mediacloud/pulls{/number}","milestones_url":"https://api.github.com/repos/berkmancenter/mediacloud/milestones{/number}","notifications_url":"https://api.github.com/repos/berkmancenter/mediacloud/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/berkmancenter/mediacloud/labels{/name}","releases_url":"https://api.github.com/repos/berkmancenter/mediacloud/releases{/id}","deployments_url":"https://api.github.com/repos/berkmancenter/mediacloud/deployments","created_at":"2013-06-26T10:59:56Z","updated_at":"2018-02-17T22:48:58Z","pushed_at":"2018-02-19T20:05:03Z","git_url":"git://github.com/berkmancenter/mediacloud.git","ssh_url":"git@github.com:berkmancenter/mediacloud.git","clone_url":"https://github.com/berkmancenter/mediacloud.git","svn_url":"https://github.com/berkmancenter/mediacloud","homepage":"http://www.mediacloud.org","size":250222,"stargazers_count":87,"watchers_count":87,"language":"HTML","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":34,"mirror_url":null,"archived":false,"open_issues_count":54,"license":{"key":"agpl-3.0","name":"GNU Affero General Public License v3.0","spdx_id":"AGPL-3.0","url":"https://api.github.com/licenses/agpl-3.0"},"forks":34,"open_issues":54,"watchers":87,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298"},"html":{"href":"https://github.com/berkmancenter/mediacloud/pull/298"},"issue":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/issues/298"},"comments":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/issues/298/comments"},"review_comments":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298/comments"},"review_comment":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/pulls/298/commits"},"statuses":{"href":"https://api.github.com/repos/berkmancenter/mediacloud/statuses/6338139437a0e04e75d9432f27137fe200bf312a"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"pypt"}}