{"action":"created","comment":{"url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/comments/65293196","id":65293196,"diff_hunk":"@@ -104,4 +121,88 @@ def findOne(self, query=None, **kwargs):\n     def validate(self, doc, **kwargs):\n         # TODO: implement\n         # raise ValidationException\n+\n+        # Validate name. This is redundant, because Folder also validates the\n+        # name, but this allows for a more specific error message.\n+        doc['name'] = doc['name'].strip()\n+        if not doc['name']:\n+            raise ValidationException('Dataset name must not be empty.', 'name')\n         return Folder.validate(self, doc, **kwargs)\n+\n+    def ingestDataset(self, uploadFolder, user, name, description, license,\n+            signature, anonymous, attribution):\n+        \"\"\"\n+        Ingest an uploaded dataset. This upload folder is expected to contain a\n+        .zip file of images and a .csv file that contains metadata about the\n+        images. The images are extracted to a new folder in the \"Phase 0\"\n+        collection and then processed.\n+        \"\"\"\n+        if not uploadFolder:\n+            raise ValidationException(\n+                'No files were uploaded.', 'uploadFolder')\n+\n+        zipFileItems = [f for f in self.model('folder').childItems(uploadFolder)\n+                        if mimetypes.guess_type(f['name'], strict=False)[0] in\n+                        ZIP_FORMATS]\n+        if not zipFileItems:\n+            raise ValidationException(\n+                'No .zip files were uploaded.', 'uploadFolder')\n+\n+        csvFileItems = [f for f in self.model('folder').childItems(uploadFolder)\n+                        if mimetypes.guess_type(f['name'], strict=False)[0] in\n+                        CSV_FORMATS]\n+        if not csvFileItems:\n+            raise ValidationException(\n+                'No .csv files were uploaded.', 'uploadFolder')\n+\n+        # Create dataset folder\n+        collection = self.model('collection').findOne({'name': 'Phase 0'})\n+        # TODO: improve error message if folder already exists\n+        datasetFolder = self.createFolder(\n+            parent=collection,\n+            name=name,\n+            description=description,\n+            parentType='collection',\n+            creator=user\n+        )\n+        datasetFolder = self.setUserAccess(\n+            doc=datasetFolder,\n+            user=user,\n+            level=AccessType.READ,\n+            save=False\n+        )\n+\n+        # Set dataset license agreement metadata\n+        datasetFolder = self.setMetadata(datasetFolder, {\n+            'signature': signature,\n+            'anonymous': anonymous,\n+            'attribution': attribution\n+        })\n+\n+        # Process zip files\n+        for item in zipFileItems:\n+            zipFiles = self.model('item').childFiles(item)\n+            for zipFile in zipFiles:\n+                handleZip(datasetFolder, user, zipFile)\n+\n+        # Process extracted images\n+        for item in self.childImages(datasetFolder):\n+            handleImage(item, user, license)\n+\n+        # Process metadata in CSV files\n+        for item in csvFileItems:\n+            csvFiles = self.model('item').childFiles(item)\n+            for csvFile in csvFiles:\n+                handleCsv(datasetFolder, user, csvFile)\n+\n+        datasetFolder = self.save(datasetFolder)\n+\n+        # Move metadata item to dataset folder. This preserves any parsing\n+        # errors that were added to the item for review.\n+        for item in csvFileItems:\n+            self.model('item').move(item, datasetFolder)\n+\n+        # Delete uploaded files\n+        self.model('folder').clean(uploadFolder)","path":"server/models/dataset.py","position":112,"original_position":112,"commit_id":"6bc475e9ca6cdbf33d556799ad7f6cb8d2a3c390","original_commit_id":"6bc475e9ca6cdbf33d556799ad7f6cb8d2a3c390","user":{"login":"brianhelba","id":1282879,"avatar_url":"https://avatars.githubusercontent.com/u/1282879?v=3","gravatar_id":"","url":"https://api.github.com/users/brianhelba","html_url":"https://github.com/brianhelba","followers_url":"https://api.github.com/users/brianhelba/followers","following_url":"https://api.github.com/users/brianhelba/following{/other_user}","gists_url":"https://api.github.com/users/brianhelba/gists{/gist_id}","starred_url":"https://api.github.com/users/brianhelba/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brianhelba/subscriptions","organizations_url":"https://api.github.com/users/brianhelba/orgs","repos_url":"https://api.github.com/users/brianhelba/repos","events_url":"https://api.github.com/users/brianhelba/events{/privacy}","received_events_url":"https://api.github.com/users/brianhelba/received_events","type":"User","site_admin":false},"body":"It might be best to comment this out for right now, to make debugging issues easier.","created_at":"2016-06-01T02:13:08Z","updated_at":"2016-06-01T02:13:08Z","html_url":"https://github.com/ImageMarkup/isic-archive/pull/58#discussion_r65293196","pull_request_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58","_links":{"self":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/comments/65293196"},"html":{"href":"https://github.com/ImageMarkup/isic-archive/pull/58#discussion_r65293196"},"pull_request":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58"}}},"pull_request":{"url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58","id":71425506,"html_url":"https://github.com/ImageMarkup/isic-archive/pull/58","diff_url":"https://github.com/ImageMarkup/isic-archive/pull/58.diff","patch_url":"https://github.com/ImageMarkup/isic-archive/pull/58.patch","issue_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/58","number":58,"state":"open","locked":false,"title":"WIP: Add dataset upload view","user":{"login":"msmolens","id":7122091,"avatar_url":"https://avatars.githubusercontent.com/u/7122091?v=3","gravatar_id":"","url":"https://api.github.com/users/msmolens","html_url":"https://github.com/msmolens","followers_url":"https://api.github.com/users/msmolens/followers","following_url":"https://api.github.com/users/msmolens/following{/other_user}","gists_url":"https://api.github.com/users/msmolens/gists{/gist_id}","starred_url":"https://api.github.com/users/msmolens/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/msmolens/subscriptions","organizations_url":"https://api.github.com/users/msmolens/orgs","repos_url":"https://api.github.com/users/msmolens/repos","events_url":"https://api.github.com/users/msmolens/events{/privacy}","received_events_url":"https://api.github.com/users/msmolens/received_events","type":"User","site_admin":false},"body":"This builds on https://github.com/ImageMarkup/isic-archive/pull/57 and adds a mockup of the dataset upload view.","created_at":"2016-05-25T16:47:50Z","updated_at":"2016-06-01T02:13:08Z","closed_at":null,"merged_at":null,"merge_commit_sha":"eb029c04f83df33de0d682d0b9bb7b0895e3b489","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58/commits","review_comments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58/comments","review_comment_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/58/comments","statuses_url":"https://api.github.com/repos/ImageMarkup/isic-archive/statuses/6bc475e9ca6cdbf33d556799ad7f6cb8d2a3c390","head":{"label":"ImageMarkup:add-dataset-upload-view","ref":"add-dataset-upload-view","sha":"6bc475e9ca6cdbf33d556799ad7f6cb8d2a3c390","user":{"login":"ImageMarkup","id":7815977,"avatar_url":"https://avatars.githubusercontent.com/u/7815977?v=3","gravatar_id":"","url":"https://api.github.com/users/ImageMarkup","html_url":"https://github.com/ImageMarkup","followers_url":"https://api.github.com/users/ImageMarkup/followers","following_url":"https://api.github.com/users/ImageMarkup/following{/other_user}","gists_url":"https://api.github.com/users/ImageMarkup/gists{/gist_id}","starred_url":"https://api.github.com/users/ImageMarkup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ImageMarkup/subscriptions","organizations_url":"https://api.github.com/users/ImageMarkup/orgs","repos_url":"https://api.github.com/users/ImageMarkup/repos","events_url":"https://api.github.com/users/ImageMarkup/events{/privacy}","received_events_url":"https://api.github.com/users/ImageMarkup/received_events","type":"Organization","site_admin":false},"repo":{"id":24576957,"name":"isic-archive","full_name":"ImageMarkup/isic-archive","owner":{"login":"ImageMarkup","id":7815977,"avatar_url":"https://avatars.githubusercontent.com/u/7815977?v=3","gravatar_id":"","url":"https://api.github.com/users/ImageMarkup","html_url":"https://github.com/ImageMarkup","followers_url":"https://api.github.com/users/ImageMarkup/followers","following_url":"https://api.github.com/users/ImageMarkup/following{/other_user}","gists_url":"https://api.github.com/users/ImageMarkup/gists{/gist_id}","starred_url":"https://api.github.com/users/ImageMarkup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ImageMarkup/subscriptions","organizations_url":"https://api.github.com/users/ImageMarkup/orgs","repos_url":"https://api.github.com/users/ImageMarkup/repos","events_url":"https://api.github.com/users/ImageMarkup/events{/privacy}","received_events_url":"https://api.github.com/users/ImageMarkup/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ImageMarkup/isic-archive","description":"","fork":false,"url":"https://api.github.com/repos/ImageMarkup/isic-archive","forks_url":"https://api.github.com/repos/ImageMarkup/isic-archive/forks","keys_url":"https://api.github.com/repos/ImageMarkup/isic-archive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ImageMarkup/isic-archive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ImageMarkup/isic-archive/teams","hooks_url":"https://api.github.com/repos/ImageMarkup/isic-archive/hooks","issue_events_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/events{/number}","events_url":"https://api.github.com/repos/ImageMarkup/isic-archive/events","assignees_url":"https://api.github.com/repos/ImageMarkup/isic-archive/assignees{/user}","branches_url":"https://api.github.com/repos/ImageMarkup/isic-archive/branches{/branch}","tags_url":"https://api.github.com/repos/ImageMarkup/isic-archive/tags","blobs_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/refs{/sha}","trees_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ImageMarkup/isic-archive/statuses/{sha}","languages_url":"https://api.github.com/repos/ImageMarkup/isic-archive/languages","stargazers_url":"https://api.github.com/repos/ImageMarkup/isic-archive/stargazers","contributors_url":"https://api.github.com/repos/ImageMarkup/isic-archive/contributors","subscribers_url":"https://api.github.com/repos/ImageMarkup/isic-archive/subscribers","subscription_url":"https://api.github.com/repos/ImageMarkup/isic-archive/subscription","commits_url":"https://api.github.com/repos/ImageMarkup/isic-archive/commits{/sha}","git_commits_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/commits{/sha}","comments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/comments{/number}","issue_comment_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/comments{/number}","contents_url":"https://api.github.com/repos/ImageMarkup/isic-archive/contents/{+path}","compare_url":"https://api.github.com/repos/ImageMarkup/isic-archive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ImageMarkup/isic-archive/merges","archive_url":"https://api.github.com/repos/ImageMarkup/isic-archive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ImageMarkup/isic-archive/downloads","issues_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues{/number}","pulls_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls{/number}","milestones_url":"https://api.github.com/repos/ImageMarkup/isic-archive/milestones{/number}","notifications_url":"https://api.github.com/repos/ImageMarkup/isic-archive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ImageMarkup/isic-archive/labels{/name}","releases_url":"https://api.github.com/repos/ImageMarkup/isic-archive/releases{/id}","deployments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/deployments","created_at":"2014-09-29T02:33:44Z","updated_at":"2016-01-14T10:21:15Z","pushed_at":"2016-05-31T22:05:57Z","git_url":"git://github.com/ImageMarkup/isic-archive.git","ssh_url":"git@github.com:ImageMarkup/isic-archive.git","clone_url":"https://github.com/ImageMarkup/isic-archive.git","svn_url":"https://github.com/ImageMarkup/isic-archive","homepage":null,"size":11135,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":31,"forks":4,"open_issues":31,"watchers":3,"default_branch":"master"}},"base":{"label":"ImageMarkup:master","ref":"master","sha":"aa2e36dc316522140c3dd7eeaad5463871dc29c4","user":{"login":"ImageMarkup","id":7815977,"avatar_url":"https://avatars.githubusercontent.com/u/7815977?v=3","gravatar_id":"","url":"https://api.github.com/users/ImageMarkup","html_url":"https://github.com/ImageMarkup","followers_url":"https://api.github.com/users/ImageMarkup/followers","following_url":"https://api.github.com/users/ImageMarkup/following{/other_user}","gists_url":"https://api.github.com/users/ImageMarkup/gists{/gist_id}","starred_url":"https://api.github.com/users/ImageMarkup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ImageMarkup/subscriptions","organizations_url":"https://api.github.com/users/ImageMarkup/orgs","repos_url":"https://api.github.com/users/ImageMarkup/repos","events_url":"https://api.github.com/users/ImageMarkup/events{/privacy}","received_events_url":"https://api.github.com/users/ImageMarkup/received_events","type":"Organization","site_admin":false},"repo":{"id":24576957,"name":"isic-archive","full_name":"ImageMarkup/isic-archive","owner":{"login":"ImageMarkup","id":7815977,"avatar_url":"https://avatars.githubusercontent.com/u/7815977?v=3","gravatar_id":"","url":"https://api.github.com/users/ImageMarkup","html_url":"https://github.com/ImageMarkup","followers_url":"https://api.github.com/users/ImageMarkup/followers","following_url":"https://api.github.com/users/ImageMarkup/following{/other_user}","gists_url":"https://api.github.com/users/ImageMarkup/gists{/gist_id}","starred_url":"https://api.github.com/users/ImageMarkup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ImageMarkup/subscriptions","organizations_url":"https://api.github.com/users/ImageMarkup/orgs","repos_url":"https://api.github.com/users/ImageMarkup/repos","events_url":"https://api.github.com/users/ImageMarkup/events{/privacy}","received_events_url":"https://api.github.com/users/ImageMarkup/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ImageMarkup/isic-archive","description":"","fork":false,"url":"https://api.github.com/repos/ImageMarkup/isic-archive","forks_url":"https://api.github.com/repos/ImageMarkup/isic-archive/forks","keys_url":"https://api.github.com/repos/ImageMarkup/isic-archive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ImageMarkup/isic-archive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ImageMarkup/isic-archive/teams","hooks_url":"https://api.github.com/repos/ImageMarkup/isic-archive/hooks","issue_events_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/events{/number}","events_url":"https://api.github.com/repos/ImageMarkup/isic-archive/events","assignees_url":"https://api.github.com/repos/ImageMarkup/isic-archive/assignees{/user}","branches_url":"https://api.github.com/repos/ImageMarkup/isic-archive/branches{/branch}","tags_url":"https://api.github.com/repos/ImageMarkup/isic-archive/tags","blobs_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/refs{/sha}","trees_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ImageMarkup/isic-archive/statuses/{sha}","languages_url":"https://api.github.com/repos/ImageMarkup/isic-archive/languages","stargazers_url":"https://api.github.com/repos/ImageMarkup/isic-archive/stargazers","contributors_url":"https://api.github.com/repos/ImageMarkup/isic-archive/contributors","subscribers_url":"https://api.github.com/repos/ImageMarkup/isic-archive/subscribers","subscription_url":"https://api.github.com/repos/ImageMarkup/isic-archive/subscription","commits_url":"https://api.github.com/repos/ImageMarkup/isic-archive/commits{/sha}","git_commits_url":"https://api.github.com/repos/ImageMarkup/isic-archive/git/commits{/sha}","comments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/comments{/number}","issue_comment_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/comments{/number}","contents_url":"https://api.github.com/repos/ImageMarkup/isic-archive/contents/{+path}","compare_url":"https://api.github.com/repos/ImageMarkup/isic-archive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ImageMarkup/isic-archive/merges","archive_url":"https://api.github.com/repos/ImageMarkup/isic-archive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ImageMarkup/isic-archive/downloads","issues_url":"https://api.github.com/repos/ImageMarkup/isic-archive/issues{/number}","pulls_url":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls{/number}","milestones_url":"https://api.github.com/repos/ImageMarkup/isic-archive/milestones{/number}","notifications_url":"https://api.github.com/repos/ImageMarkup/isic-archive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ImageMarkup/isic-archive/labels{/name}","releases_url":"https://api.github.com/repos/ImageMarkup/isic-archive/releases{/id}","deployments_url":"https://api.github.com/repos/ImageMarkup/isic-archive/deployments","created_at":"2014-09-29T02:33:44Z","updated_at":"2016-01-14T10:21:15Z","pushed_at":"2016-05-31T22:05:57Z","git_url":"git://github.com/ImageMarkup/isic-archive.git","ssh_url":"git@github.com:ImageMarkup/isic-archive.git","clone_url":"https://github.com/ImageMarkup/isic-archive.git","svn_url":"https://github.com/ImageMarkup/isic-archive","homepage":null,"size":11135,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":31,"forks":4,"open_issues":31,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58"},"html":{"href":"https://github.com/ImageMarkup/isic-archive/pull/58"},"issue":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/58"},"comments":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/issues/58/comments"},"review_comments":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58/comments"},"review_comment":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/pulls/58/commits"},"statuses":{"href":"https://api.github.com/repos/ImageMarkup/isic-archive/statuses/6bc475e9ca6cdbf33d556799ad7f6cb8d2a3c390"}}}}
{"actor":{"display_login":"brianhelba"}}