{"action":"created","comment":{"url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/comments/208596393","pull_request_review_id":144437312,"id":208596393,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwODU5NjM5Mw==","diff_hunk":"@@ -1,97 +1,81 @@\n-from contextlib import closing\n from logging import getLogger\n \n import requests\n from django.conf import settings\n-from django.utils.timezone import now\n-from django_pglocks import advisory_lock\n-from raven.contrib.django.raven_compat.models import client\n+from requests.exceptions import HTTPError\n from requests_toolbelt.multipart.encoder import MultipartEncoder\n+from requests_toolbelt.streaming_iterator import StreamingIterator\n \n-from datahub.core.exceptions import DataHubException\n-from datahub.core.utils import get_s3_client\n-from datahub.documents.models import Document\n+from .exceptions import VirusScanException\n+from .utils import get_document_by_pk\n \n logger = getLogger(__name__)\n \n \n-class S3StreamingBodyWrapper:\n-    \"\"\"S3 Object wrapper that plays nice with streamed multipart/form-data.\"\"\"\n-\n-    def __init__(self, s3_obj):\n-        \"\"\"Init wrapper, and grab interesting bits from s3 object.\"\"\"\n-        self._obj = s3_obj\n-        self._body = s3_obj['Body']\n-        self._remaining_bytes = s3_obj['ContentLength']\n-\n-    def read(self, amt=-1):\n-        \"\"\"Read given amount of bytes, and decrease remaining len.\"\"\"\n-        content = self._body.read(amt)\n-        self._remaining_bytes -= len(content)\n-\n-        return content\n-\n-    def __len__(self):\n-        \"\"\"Return remaining bytes, that have not been read yet.\n-\n-        requests-toolbelt expects this to return the number of unread bytes (rather than\n-        the total length of the stream).\n-        \"\"\"\n-        return self._remaining_bytes\n-\n-\n-def virus_scan_document(document_pk: str):\n-    \"\"\"Virus scans an uploaded document.\n-\n-    This is intended to be run in the thread pool executor. The file is streamed from S3 to the\n-    anti-virus service.\n-\n-    Any errors are logged and sent to Sentry.\n+def perform_virus_scan(document_pk: str, download_url: str):","path":"datahub/documents/av_scan.py","position":54,"original_position":54,"commit_id":"47960221d962549578c59b7ea138833a296aa72f","original_commit_id":"47960221d962549578c59b7ea138833a296aa72f","user":{"login":"marcofucci","id":178865,"node_id":"MDQ6VXNlcjE3ODg2NQ==","avatar_url":"https://avatars3.githubusercontent.com/u/178865?v=4","gravatar_id":"","url":"https://api.github.com/users/marcofucci","html_url":"https://github.com/marcofucci","followers_url":"https://api.github.com/users/marcofucci/followers","following_url":"https://api.github.com/users/marcofucci/following{/other_user}","gists_url":"https://api.github.com/users/marcofucci/gists{/gist_id}","starred_url":"https://api.github.com/users/marcofucci/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marcofucci/subscriptions","organizations_url":"https://api.github.com/users/marcofucci/orgs","repos_url":"https://api.github.com/users/marcofucci/repos","events_url":"https://api.github.com/users/marcofucci/events{/privacy}","received_events_url":"https://api.github.com/users/marcofucci/received_events","type":"User","site_admin":false},"body":"Also, you actually need `download_url` in `_download_and_scan_file` only.","created_at":"2018-08-08T14:12:38Z","updated_at":"2018-08-08T14:12:38Z","html_url":"https://github.com/uktrade/data-hub-leeloo/pull/1019#discussion_r208596393","pull_request_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/comments/208596393"},"html":{"href":"https://github.com/uktrade/data-hub-leeloo/pull/1019#discussion_r208596393"},"pull_request":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019"}},"in_reply_to_id":208595437},"pull_request":{"url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019","id":200451056,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAwNDUxMDU2","html_url":"https://github.com/uktrade/data-hub-leeloo/pull/1019","diff_url":"https://github.com/uktrade/data-hub-leeloo/pull/1019.diff","patch_url":"https://github.com/uktrade/data-hub-leeloo/pull/1019.patch","issue_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/1019","number":1019,"state":"open","locked":false,"title":"Add proposition document upload","user":{"login":"elcct","id":5889630,"node_id":"MDQ6VXNlcjU4ODk2MzA=","avatar_url":"https://avatars2.githubusercontent.com/u/5889630?v=4","gravatar_id":"","url":"https://api.github.com/users/elcct","html_url":"https://github.com/elcct","followers_url":"https://api.github.com/users/elcct/followers","following_url":"https://api.github.com/users/elcct/following{/other_user}","gists_url":"https://api.github.com/users/elcct/gists{/gist_id}","starred_url":"https://api.github.com/users/elcct/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elcct/subscriptions","organizations_url":"https://api.github.com/users/elcct/orgs","repos_url":"https://api.github.com/users/elcct/repos","events_url":"https://api.github.com/users/elcct/events{/privacy}","received_events_url":"https://api.github.com/users/elcct/received_events","type":"User","site_admin":false},"body":"Issue number: DHI-111\r\n\r\n### Description of change\r\n\r\nThis updates existing Document Upload functionality, so that it is re-usable more easily. \r\nCommon views and base model has been moved to Documents app.\r\nS3 dependent code in virus scanning has been removed.\r\n\r\nBecause we have separate AWS credentials for different type of documents I have added `bucket` field to the shared Document model, so that it could be possible to find where file has actually been uploaded.\r\n\r\nThe flow looks like this:\r\n- Client calls POST `/v3/investment/<ip-uuid>/proposition/<p-uuid>/document` with filename, gets back proposition document details with signed upload url, where file needs to be uploaded to\r\n- Once uploaded, client POST to `/v3/investment/<ip-uuid>/proposition/<p-uuid>/document/<d-uuid>/upload-callback` with empty body. This initiates virus scanning.\r\n- Once scanning has completed, the download endpoint will return signed S3 URL to the document.\r\n\r\nExisting document upload endpoints could be removed in separate PR.\r\n\r\n### Checklist\r\n\r\n* [X] Has the README been updated (if needed)?\r\n","created_at":"2018-07-10T16:15:49Z","updated_at":"2018-08-08T14:12:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4f821c5f4998fbacfa6f6c173479b58bc323ef34","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019/commits","review_comments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019/comments","review_comment_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/comments{/number}","comments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/1019/comments","statuses_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/statuses/47960221d962549578c59b7ea138833a296aa72f","head":{"label":"uktrade:feature/proposition-document-upload","ref":"feature/proposition-document-upload","sha":"47960221d962549578c59b7ea138833a296aa72f","user":{"login":"uktrade","id":15107524,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTA3NTI0","avatar_url":"https://avatars2.githubusercontent.com/u/15107524?v=4","gravatar_id":"","url":"https://api.github.com/users/uktrade","html_url":"https://github.com/uktrade","followers_url":"https://api.github.com/users/uktrade/followers","following_url":"https://api.github.com/users/uktrade/following{/other_user}","gists_url":"https://api.github.com/users/uktrade/gists{/gist_id}","starred_url":"https://api.github.com/users/uktrade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uktrade/subscriptions","organizations_url":"https://api.github.com/users/uktrade/orgs","repos_url":"https://api.github.com/users/uktrade/repos","events_url":"https://api.github.com/users/uktrade/events{/privacy}","received_events_url":"https://api.github.com/users/uktrade/received_events","type":"Organization","site_admin":false},"repo":{"id":73379434,"node_id":"MDEwOlJlcG9zaXRvcnk3MzM3OTQzNA==","name":"data-hub-leeloo","full_name":"uktrade/data-hub-leeloo","owner":{"login":"uktrade","id":15107524,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTA3NTI0","avatar_url":"https://avatars2.githubusercontent.com/u/15107524?v=4","gravatar_id":"","url":"https://api.github.com/users/uktrade","html_url":"https://github.com/uktrade","followers_url":"https://api.github.com/users/uktrade/followers","following_url":"https://api.github.com/users/uktrade/following{/other_user}","gists_url":"https://api.github.com/users/uktrade/gists{/gist_id}","starred_url":"https://api.github.com/users/uktrade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uktrade/subscriptions","organizations_url":"https://api.github.com/users/uktrade/orgs","repos_url":"https://api.github.com/users/uktrade/repos","events_url":"https://api.github.com/users/uktrade/events{/privacy}","received_events_url":"https://api.github.com/users/uktrade/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uktrade/data-hub-leeloo","description":"Django API for Data Hub frontend","fork":false,"url":"https://api.github.com/repos/uktrade/data-hub-leeloo","forks_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/forks","keys_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/teams","hooks_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/hooks","issue_events_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/events{/number}","events_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/events","assignees_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/assignees{/user}","branches_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/branches{/branch}","tags_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/tags","blobs_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/refs{/sha}","trees_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/statuses/{sha}","languages_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/languages","stargazers_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/stargazers","contributors_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/contributors","subscribers_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/subscribers","subscription_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/subscription","commits_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/commits{/sha}","git_commits_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/commits{/sha}","comments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/comments{/number}","issue_comment_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/comments{/number}","contents_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/contents/{+path}","compare_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/merges","archive_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/downloads","issues_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues{/number}","pulls_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls{/number}","milestones_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/milestones{/number}","notifications_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/labels{/name}","releases_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/releases{/id}","deployments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/deployments","created_at":"2016-11-10T12:21:01Z","updated_at":"2018-08-08T13:26:30Z","pushed_at":"2018-08-08T13:26:31Z","git_url":"git://github.com/uktrade/data-hub-leeloo.git","ssh_url":"git@github.com:uktrade/data-hub-leeloo.git","clone_url":"https://github.com/uktrade/data-hub-leeloo.git","svn_url":"https://github.com/uktrade/data-hub-leeloo","homepage":null,"size":5596,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":9,"license":null,"forks":0,"open_issues":9,"watchers":1,"default_branch":"develop"}},"base":{"label":"uktrade:develop","ref":"develop","sha":"694a69d9d1018fb22634497df62a75110ec42d6c","user":{"login":"uktrade","id":15107524,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTA3NTI0","avatar_url":"https://avatars2.githubusercontent.com/u/15107524?v=4","gravatar_id":"","url":"https://api.github.com/users/uktrade","html_url":"https://github.com/uktrade","followers_url":"https://api.github.com/users/uktrade/followers","following_url":"https://api.github.com/users/uktrade/following{/other_user}","gists_url":"https://api.github.com/users/uktrade/gists{/gist_id}","starred_url":"https://api.github.com/users/uktrade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uktrade/subscriptions","organizations_url":"https://api.github.com/users/uktrade/orgs","repos_url":"https://api.github.com/users/uktrade/repos","events_url":"https://api.github.com/users/uktrade/events{/privacy}","received_events_url":"https://api.github.com/users/uktrade/received_events","type":"Organization","site_admin":false},"repo":{"id":73379434,"node_id":"MDEwOlJlcG9zaXRvcnk3MzM3OTQzNA==","name":"data-hub-leeloo","full_name":"uktrade/data-hub-leeloo","owner":{"login":"uktrade","id":15107524,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTA3NTI0","avatar_url":"https://avatars2.githubusercontent.com/u/15107524?v=4","gravatar_id":"","url":"https://api.github.com/users/uktrade","html_url":"https://github.com/uktrade","followers_url":"https://api.github.com/users/uktrade/followers","following_url":"https://api.github.com/users/uktrade/following{/other_user}","gists_url":"https://api.github.com/users/uktrade/gists{/gist_id}","starred_url":"https://api.github.com/users/uktrade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uktrade/subscriptions","organizations_url":"https://api.github.com/users/uktrade/orgs","repos_url":"https://api.github.com/users/uktrade/repos","events_url":"https://api.github.com/users/uktrade/events{/privacy}","received_events_url":"https://api.github.com/users/uktrade/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uktrade/data-hub-leeloo","description":"Django API for Data Hub frontend","fork":false,"url":"https://api.github.com/repos/uktrade/data-hub-leeloo","forks_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/forks","keys_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/teams","hooks_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/hooks","issue_events_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/events{/number}","events_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/events","assignees_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/assignees{/user}","branches_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/branches{/branch}","tags_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/tags","blobs_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/refs{/sha}","trees_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/statuses/{sha}","languages_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/languages","stargazers_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/stargazers","contributors_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/contributors","subscribers_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/subscribers","subscription_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/subscription","commits_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/commits{/sha}","git_commits_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/git/commits{/sha}","comments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/comments{/number}","issue_comment_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/comments{/number}","contents_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/contents/{+path}","compare_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/merges","archive_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/downloads","issues_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues{/number}","pulls_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls{/number}","milestones_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/milestones{/number}","notifications_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/labels{/name}","releases_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/releases{/id}","deployments_url":"https://api.github.com/repos/uktrade/data-hub-leeloo/deployments","created_at":"2016-11-10T12:21:01Z","updated_at":"2018-08-08T13:26:30Z","pushed_at":"2018-08-08T13:26:31Z","git_url":"git://github.com/uktrade/data-hub-leeloo.git","ssh_url":"git@github.com:uktrade/data-hub-leeloo.git","clone_url":"https://github.com/uktrade/data-hub-leeloo.git","svn_url":"https://github.com/uktrade/data-hub-leeloo","homepage":null,"size":5596,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":9,"license":null,"forks":0,"open_issues":9,"watchers":1,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019"},"html":{"href":"https://github.com/uktrade/data-hub-leeloo/pull/1019"},"issue":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/1019"},"comments":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/issues/1019/comments"},"review_comments":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019/comments"},"review_comment":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/pulls/1019/commits"},"statuses":{"href":"https://api.github.com/repos/uktrade/data-hub-leeloo/statuses/47960221d962549578c59b7ea138833a296aa72f"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"marcofucci"}}