{"action":"created","comment":{"url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/comments/198924093","pull_request_review_id":132936468,"id":198924093,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDE5ODkyNDA5Mw==","diff_hunk":"@@ -17,88 +18,132 @@\n     'username': re.compile('(?:@)([A-Za-z0-9_](?:(?:[A-Za-z0-9_]|(?:\\.(?!\\.))){0,28}(?:[A-Za-z0-9_]))?)'),\n }\n \n+# In case Instagram switches it up on us\n+IMG_XPATH = '//img[@alt]'\n \n-def scrape_instagram_tag(tag: str, total_count: int=50, existing: set=None):\n+def send_scrape_request(insta_url: str, total_count: int=50, existing: set=None, short_circuit: bool=False):\n     \"\"\"\n-    Scrape and yield recently tagged instagram photos.\n+        :param insta_url:\n+            Instagram url to scrape\n+        :param total_count:\n+            Total amount of images to scrape\n+        :param existing:\n+            URLs to skip\n+        :param short_circuit:\n+            Whether or not to short_circuit total_count loop \n+            \n+    Yields url, captions, hashtags, and mentions for provided insta url\n     \"\"\"\n-    url = f'https://www.instagram.com/explore/tags/{tag}'\n     session = HTMLSession()\n-    req = session.get(url)\n+    req = session.get(insta_url)\n \n-    imgs = set(existing)\n+    imgs = set()\n     count = 0\n     page = 0\n-\n-    while count <= total_count:\n+    while count < total_count:\n         req.html.render(scrolldown=page)\n-        images = req.html.xpath('//img[@alt]')\n+        images = req.html.xpath(IMG_XPATH)  \n         page += 1\n         for image in images:\n-            if count > total_count:\n+            if count >= total_count:\n                 break\n             try:\n                 url, caption = image.attrs['src'], image.attrs['alt']\n-            except:\n-                pass\n-            else:\n-                if url in imgs:\n-                    continue\n-                imgs.add(url)\n-                hashtags = set(REGEXES['hashtag'].findall(caption))\n-                mentions = set(REGEXES['username'].findall(caption))\n-                count += 1\n-                yield url, caption, hashtags, mentions\n-\n-\n-def scrape_instagram(tags: List[str], total_count: int=50, existing: set=None):\n+            except Exception as e:\n+                print(e)\n+            if url in imgs:\n+                # Short-circuit if user has less photos than the total_count\n+                if short_circuit:\n+                    if len(images) < total_count: \n+                        total_count = 0\n+                        break\n+                continue\n+            imgs.add(url)\n+            hashtags = set(REGEXES['hashtag'].findall(caption))\n+            mentions = set(REGEXES['username'].findall(caption))\n+            count += 1\n+            yield url, caption, hashtags, mentions\n+\n+\n+\n+def scrape_instagram(target: str, total_count: int=50, existing: set=None, mode: str='tags'):\n     \"\"\"\n-    :param tags:\n-        List of tags that need to be scraped.\n+    :param targets:\n+        List of targets that need to be scraped.\n     :param total_count:\n         Total number of images to be scraped.\n+    :param existing:\n+        URLs to skip\n+    :param mode","path":"instagram_scraper.py","position":89,"original_position":89,"commit_id":"ab2004ecb8873bedb6e791b87be9b064b97513c3","original_commit_id":"ab2004ecb8873bedb6e791b87be9b064b97513c3","user":{"login":"meetmangukiya","id":7620533,"node_id":"MDQ6VXNlcjc2MjA1MzM=","avatar_url":"https://avatars0.githubusercontent.com/u/7620533?v=4","gravatar_id":"","url":"https://api.github.com/users/meetmangukiya","html_url":"https://github.com/meetmangukiya","followers_url":"https://api.github.com/users/meetmangukiya/followers","following_url":"https://api.github.com/users/meetmangukiya/following{/other_user}","gists_url":"https://api.github.com/users/meetmangukiya/gists{/gist_id}","starred_url":"https://api.github.com/users/meetmangukiya/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meetmangukiya/subscriptions","organizations_url":"https://api.github.com/users/meetmangukiya/orgs","repos_url":"https://api.github.com/users/meetmangukiya/repos","events_url":"https://api.github.com/users/meetmangukiya/events{/privacy}","received_events_url":"https://api.github.com/users/meetmangukiya/received_events","type":"User","site_admin":false},"body":"add a colon after `mode`","created_at":"2018-06-28T17:32:05Z","updated_at":"2018-06-28T17:59:48Z","html_url":"https://github.com/meetmangukiya/instagram-scraper/pull/12#discussion_r198924093","pull_request_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/comments/198924093"},"html":{"href":"https://github.com/meetmangukiya/instagram-scraper/pull/12#discussion_r198924093"},"pull_request":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12"}}},"pull_request":{"url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12","id":197911426,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk3OTExNDI2","html_url":"https://github.com/meetmangukiya/instagram-scraper/pull/12","diff_url":"https://github.com/meetmangukiya/instagram-scraper/pull/12.diff","patch_url":"https://github.com/meetmangukiya/instagram-scraper/pull/12.patch","issue_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/12","number":12,"state":"open","locked":false,"title":"Add ability to scrape user pages","user":{"login":"xraymemory","id":970291,"node_id":"MDQ6VXNlcjk3MDI5MQ==","avatar_url":"https://avatars1.githubusercontent.com/u/970291?v=4","gravatar_id":"","url":"https://api.github.com/users/xraymemory","html_url":"https://github.com/xraymemory","followers_url":"https://api.github.com/users/xraymemory/followers","following_url":"https://api.github.com/users/xraymemory/following{/other_user}","gists_url":"https://api.github.com/users/xraymemory/gists{/gist_id}","starred_url":"https://api.github.com/users/xraymemory/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xraymemory/subscriptions","organizations_url":"https://api.github.com/users/xraymemory/orgs","repos_url":"https://api.github.com/users/xraymemory/repos","events_url":"https://api.github.com/users/xraymemory/events{/privacy}","received_events_url":"https://api.github.com/users/xraymemory/received_events","type":"User","site_admin":false},"body":"Refactored the code so that you can specify both tags and users to be scraped. Also fixed some off by one errors and added more function documentation. ","created_at":"2018-06-28T04:01:06Z","updated_at":"2018-06-28T17:59:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"63185157a1ef33656c8ababa7595e29b3a0ab6fb","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12/commits","review_comments_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12/comments","review_comment_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/comments{/number}","comments_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/12/comments","statuses_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/statuses/ab2004ecb8873bedb6e791b87be9b064b97513c3","head":{"label":"xraymemory:patch-1","ref":"patch-1","sha":"ab2004ecb8873bedb6e791b87be9b064b97513c3","user":{"login":"xraymemory","id":970291,"node_id":"MDQ6VXNlcjk3MDI5MQ==","avatar_url":"https://avatars1.githubusercontent.com/u/970291?v=4","gravatar_id":"","url":"https://api.github.com/users/xraymemory","html_url":"https://github.com/xraymemory","followers_url":"https://api.github.com/users/xraymemory/followers","following_url":"https://api.github.com/users/xraymemory/following{/other_user}","gists_url":"https://api.github.com/users/xraymemory/gists{/gist_id}","starred_url":"https://api.github.com/users/xraymemory/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xraymemory/subscriptions","organizations_url":"https://api.github.com/users/xraymemory/orgs","repos_url":"https://api.github.com/users/xraymemory/repos","events_url":"https://api.github.com/users/xraymemory/events{/privacy}","received_events_url":"https://api.github.com/users/xraymemory/received_events","type":"User","site_admin":false},"repo":{"id":138962528,"node_id":"MDEwOlJlcG9zaXRvcnkxMzg5NjI1Mjg=","name":"instagram-scraper","full_name":"xraymemory/instagram-scraper","owner":{"login":"xraymemory","id":970291,"node_id":"MDQ6VXNlcjk3MDI5MQ==","avatar_url":"https://avatars1.githubusercontent.com/u/970291?v=4","gravatar_id":"","url":"https://api.github.com/users/xraymemory","html_url":"https://github.com/xraymemory","followers_url":"https://api.github.com/users/xraymemory/followers","following_url":"https://api.github.com/users/xraymemory/following{/other_user}","gists_url":"https://api.github.com/users/xraymemory/gists{/gist_id}","starred_url":"https://api.github.com/users/xraymemory/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xraymemory/subscriptions","organizations_url":"https://api.github.com/users/xraymemory/orgs","repos_url":"https://api.github.com/users/xraymemory/repos","events_url":"https://api.github.com/users/xraymemory/events{/privacy}","received_events_url":"https://api.github.com/users/xraymemory/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/xraymemory/instagram-scraper","description":"Scrape the Instagram frontend. Inspired from twitter-scraper by @kennethreitz.","fork":true,"url":"https://api.github.com/repos/xraymemory/instagram-scraper","forks_url":"https://api.github.com/repos/xraymemory/instagram-scraper/forks","keys_url":"https://api.github.com/repos/xraymemory/instagram-scraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/xraymemory/instagram-scraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/xraymemory/instagram-scraper/teams","hooks_url":"https://api.github.com/repos/xraymemory/instagram-scraper/hooks","issue_events_url":"https://api.github.com/repos/xraymemory/instagram-scraper/issues/events{/number}","events_url":"https://api.github.com/repos/xraymemory/instagram-scraper/events","assignees_url":"https://api.github.com/repos/xraymemory/instagram-scraper/assignees{/user}","branches_url":"https://api.github.com/repos/xraymemory/instagram-scraper/branches{/branch}","tags_url":"https://api.github.com/repos/xraymemory/instagram-scraper/tags","blobs_url":"https://api.github.com/repos/xraymemory/instagram-scraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/xraymemory/instagram-scraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/xraymemory/instagram-scraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/xraymemory/instagram-scraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/xraymemory/instagram-scraper/statuses/{sha}","languages_url":"https://api.github.com/repos/xraymemory/instagram-scraper/languages","stargazers_url":"https://api.github.com/repos/xraymemory/instagram-scraper/stargazers","contributors_url":"https://api.github.com/repos/xraymemory/instagram-scraper/contributors","subscribers_url":"https://api.github.com/repos/xraymemory/instagram-scraper/subscribers","subscription_url":"https://api.github.com/repos/xraymemory/instagram-scraper/subscription","commits_url":"https://api.github.com/repos/xraymemory/instagram-scraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/xraymemory/instagram-scraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/xraymemory/instagram-scraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/xraymemory/instagram-scraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/xraymemory/instagram-scraper/contents/{+path}","compare_url":"https://api.github.com/repos/xraymemory/instagram-scraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/xraymemory/instagram-scraper/merges","archive_url":"https://api.github.com/repos/xraymemory/instagram-scraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/xraymemory/instagram-scraper/downloads","issues_url":"https://api.github.com/repos/xraymemory/instagram-scraper/issues{/number}","pulls_url":"https://api.github.com/repos/xraymemory/instagram-scraper/pulls{/number}","milestones_url":"https://api.github.com/repos/xraymemory/instagram-scraper/milestones{/number}","notifications_url":"https://api.github.com/repos/xraymemory/instagram-scraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/xraymemory/instagram-scraper/labels{/name}","releases_url":"https://api.github.com/repos/xraymemory/instagram-scraper/releases{/id}","deployments_url":"https://api.github.com/repos/xraymemory/instagram-scraper/deployments","created_at":"2018-06-28T03:38:11Z","updated_at":"2018-06-28T04:04:00Z","pushed_at":"2018-06-28T04:03:59Z","git_url":"git://github.com/xraymemory/instagram-scraper.git","ssh_url":"git@github.com:xraymemory/instagram-scraper.git","clone_url":"https://github.com/xraymemory/instagram-scraper.git","svn_url":"https://github.com/xraymemory/instagram-scraper","homepage":"","size":27,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"meetmangukiya:master","ref":"master","sha":"f1080ead623e385f9192c87ee3fc70ed2c9992e7","user":{"login":"meetmangukiya","id":7620533,"node_id":"MDQ6VXNlcjc2MjA1MzM=","avatar_url":"https://avatars0.githubusercontent.com/u/7620533?v=4","gravatar_id":"","url":"https://api.github.com/users/meetmangukiya","html_url":"https://github.com/meetmangukiya","followers_url":"https://api.github.com/users/meetmangukiya/followers","following_url":"https://api.github.com/users/meetmangukiya/following{/other_user}","gists_url":"https://api.github.com/users/meetmangukiya/gists{/gist_id}","starred_url":"https://api.github.com/users/meetmangukiya/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meetmangukiya/subscriptions","organizations_url":"https://api.github.com/users/meetmangukiya/orgs","repos_url":"https://api.github.com/users/meetmangukiya/repos","events_url":"https://api.github.com/users/meetmangukiya/events{/privacy}","received_events_url":"https://api.github.com/users/meetmangukiya/received_events","type":"User","site_admin":false},"repo":{"id":138404520,"node_id":"MDEwOlJlcG9zaXRvcnkxMzg0MDQ1MjA=","name":"instagram-scraper","full_name":"meetmangukiya/instagram-scraper","owner":{"login":"meetmangukiya","id":7620533,"node_id":"MDQ6VXNlcjc2MjA1MzM=","avatar_url":"https://avatars0.githubusercontent.com/u/7620533?v=4","gravatar_id":"","url":"https://api.github.com/users/meetmangukiya","html_url":"https://github.com/meetmangukiya","followers_url":"https://api.github.com/users/meetmangukiya/followers","following_url":"https://api.github.com/users/meetmangukiya/following{/other_user}","gists_url":"https://api.github.com/users/meetmangukiya/gists{/gist_id}","starred_url":"https://api.github.com/users/meetmangukiya/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/meetmangukiya/subscriptions","organizations_url":"https://api.github.com/users/meetmangukiya/orgs","repos_url":"https://api.github.com/users/meetmangukiya/repos","events_url":"https://api.github.com/users/meetmangukiya/events{/privacy}","received_events_url":"https://api.github.com/users/meetmangukiya/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/meetmangukiya/instagram-scraper","description":"Scrape the Instagram frontend. Inspired from twitter-scraper by @kennethreitz.","fork":false,"url":"https://api.github.com/repos/meetmangukiya/instagram-scraper","forks_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/forks","keys_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/teams","hooks_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/hooks","issue_events_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/events{/number}","events_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/events","assignees_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/assignees{/user}","branches_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/branches{/branch}","tags_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/tags","blobs_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/statuses/{sha}","languages_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/languages","stargazers_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/stargazers","contributors_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/contributors","subscribers_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/subscribers","subscription_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/subscription","commits_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/contents/{+path}","compare_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/merges","archive_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/downloads","issues_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues{/number}","pulls_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls{/number}","milestones_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/milestones{/number}","notifications_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/labels{/name}","releases_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/releases{/id}","deployments_url":"https://api.github.com/repos/meetmangukiya/instagram-scraper/deployments","created_at":"2018-06-23T13:45:59Z","updated_at":"2018-06-28T17:10:25Z","pushed_at":"2018-06-28T17:16:01Z","git_url":"git://github.com/meetmangukiya/instagram-scraper.git","ssh_url":"git@github.com:meetmangukiya/instagram-scraper.git","clone_url":"https://github.com/meetmangukiya/instagram-scraper.git","svn_url":"https://github.com/meetmangukiya/instagram-scraper","homepage":"","size":9,"stargazers_count":652,"watchers_count":652,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":49,"mirror_url":null,"archived":false,"open_issues_count":8,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":49,"open_issues":8,"watchers":652,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12"},"html":{"href":"https://github.com/meetmangukiya/instagram-scraper/pull/12"},"issue":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/12"},"comments":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/issues/12/comments"},"review_comments":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12/comments"},"review_comment":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/pulls/12/commits"},"statuses":{"href":"https://api.github.com/repos/meetmangukiya/instagram-scraper/statuses/ab2004ecb8873bedb6e791b87be9b064b97513c3"}},"author_association":"NONE"}}
{"actor":{"display_login":"meetmangukiya"}}