{"action":"created","comment":{"url":"https://api.github.com/repos/aodn/python-aodndata/pulls/comments/189141951","pull_request_review_id":121263078,"id":189141951,"diff_hunk":"@@ -0,0 +1,119 @@\n+from aodncore.pipeline import HandlerBase, PipelineFile, PipelineFilePublishType, PipelineFileCheckType, FileType\n+from aodncore.pipeline import PipelineFileCollection\n+from aodncore.pipeline.exceptions import InvalidInputFileError\n+from aodndata.soop.soop_ba_classifier import SoopBaFileClassifier\n+from aodncore.pipeline.statequery import StateQuery\n+import re\n+import os\n+\n+\n+class SoopBaHandler(HandlerBase):\n+    \"\"\"\n+    Handler for data from IMOS SOOP BA\n+    It handles the following file types:\n+         * ZIP;\n+         * NetCDF;\n+    \"\"\"\n+    def __init__(self, *args, **kwargs):\n+        super(SoopBaHandler, self).__init__(*args, **kwargs)\n+        self.allowed_extensions = ['.nc', '.zip']\n+\n+    def preprocess(self):\n+        \"\"\" Preprocessing of Zip archive and NetCDF files\n+            Preprocessing consist in setting the destination path AND deleting previous version files\n+            - Zip contains netcdf , images ,text, doc, or xml file and raw file to archive\n+             dest_path is generated based on info stored in FV01 NetCDF file.\n+             update check_type and publish_type according to destination :\n+             raw files :  move to archive =>publish_type property to 'archive'\n+            - text, doc, xml, images: basic checks\n+              uploaded to S3 => set check_type and publish_type attributesge accordingly\n+        \"\"\"\n+        if self.file_type is FileType.ZIP:\n+            netcdf = self.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF)\n+            # if uploaded file is ZIP check that ZIP contains a NetCDF\n+            if len(netcdf) != 1:\n+                raise InvalidInputFileError(\n+                    \"Expecting one netCDF file in ZIP archive '{zip}'\".format(zip=os.path.basename(self.input_file))\n+                )\n+\n+            # first process the NetCDF file to set the destination path for the file collection.\n+            nc = netcdf[0]\n+            nc.check_type = PipelineFileCheckType.NC_COMPLIANCE_CHECK\n+            destination = SoopBaFileClassifier.dest_path(nc)\n+            nc.dest_path = os.path.join(destination, nc.name)\n+\n+            #  delete previous version base on filetype/extension\n+            # netcdf file have to be deleted since filename contains creation date=>never overwritten\n+            results = self.state_query.query_storage(destination)\n+            if results:\n+                get_previous_version(self, results, nc.name, destination)\n+\n+            non_nc_files = PipelineFileCollection(f for f in self.file_collection if f.file_type is not FileType.NETCDF)\n+            for non_nc in non_nc_files:\n+                non_nc.check_type = PipelineFileCheckType.FORMAT_CHECK\n+                if non_nc.extension in ['.ek5', '.out', '.raw']:\n+                    non_nc.publish_type = PipelineFilePublishType.ARCHIVE_ONLY\n+                    dest_archive = SoopBaFileClassifier.archive_path(nc)\n+                    non_nc.archive_path = os.path.join(dest_archive, non_nc.name)\n+                else:\n+                    non_nc.publish_type = PipelineFilePublishType.UPLOAD_ONLY\n+                    non_nc.dest_path = os.path.join(destination, non_nc.name)\n+                    if results:\n+                        get_previous_version(self, results, non_nc.name)\n+\n+        else:  # singe netcdf file\n+            netcdf = self.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF)\n+            nc = netcdf[0]\n+            nc.check_type = PipelineFileCheckType.NC_COMPLIANCE_CHECK\n+            destination = SoopBaFileClassifier.dest_path(nc)\n+            nc.dest_path = os.path.join(destination, nc.name)\n+\n+            #  delete previous version base on filetype/extension\n+            # netcdf file have to be deleted since filename contains creation date=>never overwritten\n+            results = self.state_query.query_storage(destination)\n+            if results:\n+                get_previous_version(self, results, nc.name, destination)\n+\n+\n+    dest_path = SoopBaFileClassifier.dest_path\n+    archive_path = SoopBaFileClassifier.archive_path\n+\n+","path":"aodndata/soop/soop_ba.py","position":81,"original_position":81,"commit_id":"eb0970e1e6615b3b0cfb2a85b4c679e9bc668519","original_commit_id":"eb0970e1e6615b3b0cfb2a85b4c679e9bc668519","user":{"login":"lbesnard","id":2382814,"avatar_url":"https://avatars2.githubusercontent.com/u/2382814?v=4","gravatar_id":"","url":"https://api.github.com/users/lbesnard","html_url":"https://github.com/lbesnard","followers_url":"https://api.github.com/users/lbesnard/followers","following_url":"https://api.github.com/users/lbesnard/following{/other_user}","gists_url":"https://api.github.com/users/lbesnard/gists{/gist_id}","starred_url":"https://api.github.com/users/lbesnard/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lbesnard/subscriptions","organizations_url":"https://api.github.com/users/lbesnard/orgs","repos_url":"https://api.github.com/users/lbesnard/repos","events_url":"https://api.github.com/users/lbesnard/events{/privacy}","received_events_url":"https://api.github.com/users/lbesnard/received_events","type":"User","site_admin":false},"body":"one line only","created_at":"2018-05-18T01:12:03Z","updated_at":"2018-05-18T01:14:26Z","html_url":"https://github.com/aodn/python-aodndata/pull/32#discussion_r189141951","pull_request_url":"https://api.github.com/repos/aodn/python-aodndata/pulls/32","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/comments/189141951"},"html":{"href":"https://github.com/aodn/python-aodndata/pull/32#discussion_r189141951"},"pull_request":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/32"}}},"pull_request":{"url":"https://api.github.com/repos/aodn/python-aodndata/pulls/32","id":188627629,"html_url":"https://github.com/aodn/python-aodndata/pull/32","diff_url":"https://github.com/aodn/python-aodndata/pull/32.diff","patch_url":"https://github.com/aodn/python-aodndata/pull/32.patch","issue_url":"https://api.github.com/repos/aodn/python-aodndata/issues/32","number":32,"state":"open","locked":false,"title":"SOOP-BA pipeline V2 and unittest","user":{"login":"bpasquer","id":3733612,"avatar_url":"https://avatars2.githubusercontent.com/u/3733612?v=4","gravatar_id":"","url":"https://api.github.com/users/bpasquer","html_url":"https://github.com/bpasquer","followers_url":"https://api.github.com/users/bpasquer/followers","following_url":"https://api.github.com/users/bpasquer/following{/other_user}","gists_url":"https://api.github.com/users/bpasquer/gists{/gist_id}","starred_url":"https://api.github.com/users/bpasquer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bpasquer/subscriptions","organizations_url":"https://api.github.com/users/bpasquer/orgs","repos_url":"https://api.github.com/users/bpasquer/repos","events_url":"https://api.github.com/users/bpasquer/events{/privacy}","received_events_url":"https://api.github.com/users/bpasquer/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-05-17T05:33:01Z","updated_at":"2018-05-18T01:14:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"33bf6efec2b55d69cbf2616951f1e71863c48a98","assignee":null,"assignees":[],"requested_reviewers":[{"login":"anguss00","id":5964166,"avatar_url":"https://avatars2.githubusercontent.com/u/5964166?v=4","gravatar_id":"","url":"https://api.github.com/users/anguss00","html_url":"https://github.com/anguss00","followers_url":"https://api.github.com/users/anguss00/followers","following_url":"https://api.github.com/users/anguss00/following{/other_user}","gists_url":"https://api.github.com/users/anguss00/gists{/gist_id}","starred_url":"https://api.github.com/users/anguss00/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anguss00/subscriptions","organizations_url":"https://api.github.com/users/anguss00/orgs","repos_url":"https://api.github.com/users/anguss00/repos","events_url":"https://api.github.com/users/anguss00/events{/privacy}","received_events_url":"https://api.github.com/users/anguss00/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/aodn/python-aodndata/pulls/32/commits","review_comments_url":"https://api.github.com/repos/aodn/python-aodndata/pulls/32/comments","review_comment_url":"https://api.github.com/repos/aodn/python-aodndata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/aodn/python-aodndata/issues/32/comments","statuses_url":"https://api.github.com/repos/aodn/python-aodndata/statuses/eb0970e1e6615b3b0cfb2a85b4c679e9bc668519","head":{"label":"aodn:soop_ba","ref":"soop_ba","sha":"eb0970e1e6615b3b0cfb2a85b4c679e9bc668519","user":{"login":"aodn","id":2802166,"avatar_url":"https://avatars2.githubusercontent.com/u/2802166?v=4","gravatar_id":"","url":"https://api.github.com/users/aodn","html_url":"https://github.com/aodn","followers_url":"https://api.github.com/users/aodn/followers","following_url":"https://api.github.com/users/aodn/following{/other_user}","gists_url":"https://api.github.com/users/aodn/gists{/gist_id}","starred_url":"https://api.github.com/users/aodn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aodn/subscriptions","organizations_url":"https://api.github.com/users/aodn/orgs","repos_url":"https://api.github.com/users/aodn/repos","events_url":"https://api.github.com/users/aodn/events{/privacy}","received_events_url":"https://api.github.com/users/aodn/received_events","type":"Organization","site_admin":false},"repo":{"id":109786059,"name":"python-aodndata","full_name":"aodn/python-aodndata","owner":{"login":"aodn","id":2802166,"avatar_url":"https://avatars2.githubusercontent.com/u/2802166?v=4","gravatar_id":"","url":"https://api.github.com/users/aodn","html_url":"https://github.com/aodn","followers_url":"https://api.github.com/users/aodn/followers","following_url":"https://api.github.com/users/aodn/following{/other_user}","gists_url":"https://api.github.com/users/aodn/gists{/gist_id}","starred_url":"https://api.github.com/users/aodn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aodn/subscriptions","organizations_url":"https://api.github.com/users/aodn/orgs","repos_url":"https://api.github.com/users/aodn/repos","events_url":"https://api.github.com/users/aodn/events{/privacy}","received_events_url":"https://api.github.com/users/aodn/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/aodn/python-aodndata","description":null,"fork":false,"url":"https://api.github.com/repos/aodn/python-aodndata","forks_url":"https://api.github.com/repos/aodn/python-aodndata/forks","keys_url":"https://api.github.com/repos/aodn/python-aodndata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aodn/python-aodndata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aodn/python-aodndata/teams","hooks_url":"https://api.github.com/repos/aodn/python-aodndata/hooks","issue_events_url":"https://api.github.com/repos/aodn/python-aodndata/issues/events{/number}","events_url":"https://api.github.com/repos/aodn/python-aodndata/events","assignees_url":"https://api.github.com/repos/aodn/python-aodndata/assignees{/user}","branches_url":"https://api.github.com/repos/aodn/python-aodndata/branches{/branch}","tags_url":"https://api.github.com/repos/aodn/python-aodndata/tags","blobs_url":"https://api.github.com/repos/aodn/python-aodndata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aodn/python-aodndata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aodn/python-aodndata/git/refs{/sha}","trees_url":"https://api.github.com/repos/aodn/python-aodndata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aodn/python-aodndata/statuses/{sha}","languages_url":"https://api.github.com/repos/aodn/python-aodndata/languages","stargazers_url":"https://api.github.com/repos/aodn/python-aodndata/stargazers","contributors_url":"https://api.github.com/repos/aodn/python-aodndata/contributors","subscribers_url":"https://api.github.com/repos/aodn/python-aodndata/subscribers","subscription_url":"https://api.github.com/repos/aodn/python-aodndata/subscription","commits_url":"https://api.github.com/repos/aodn/python-aodndata/commits{/sha}","git_commits_url":"https://api.github.com/repos/aodn/python-aodndata/git/commits{/sha}","comments_url":"https://api.github.com/repos/aodn/python-aodndata/comments{/number}","issue_comment_url":"https://api.github.com/repos/aodn/python-aodndata/issues/comments{/number}","contents_url":"https://api.github.com/repos/aodn/python-aodndata/contents/{+path}","compare_url":"https://api.github.com/repos/aodn/python-aodndata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aodn/python-aodndata/merges","archive_url":"https://api.github.com/repos/aodn/python-aodndata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aodn/python-aodndata/downloads","issues_url":"https://api.github.com/repos/aodn/python-aodndata/issues{/number}","pulls_url":"https://api.github.com/repos/aodn/python-aodndata/pulls{/number}","milestones_url":"https://api.github.com/repos/aodn/python-aodndata/milestones{/number}","notifications_url":"https://api.github.com/repos/aodn/python-aodndata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aodn/python-aodndata/labels{/name}","releases_url":"https://api.github.com/repos/aodn/python-aodndata/releases{/id}","deployments_url":"https://api.github.com/repos/aodn/python-aodndata/deployments","created_at":"2017-11-07T04:17:30Z","updated_at":"2018-05-17T22:15:19Z","pushed_at":"2018-05-17T22:15:21Z","git_url":"git://github.com/aodn/python-aodndata.git","ssh_url":"git@github.com:aodn/python-aodndata.git","clone_url":"https://github.com/aodn/python-aodndata.git","svn_url":"https://github.com/aodn/python-aodndata","homepage":null,"size":237323,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":3,"license":null,"forks":0,"open_issues":3,"watchers":0,"default_branch":"master"}},"base":{"label":"aodn:master","ref":"master","sha":"214be7255ec3def45f7fae6b0aaad4fefd8171a3","user":{"login":"aodn","id":2802166,"avatar_url":"https://avatars2.githubusercontent.com/u/2802166?v=4","gravatar_id":"","url":"https://api.github.com/users/aodn","html_url":"https://github.com/aodn","followers_url":"https://api.github.com/users/aodn/followers","following_url":"https://api.github.com/users/aodn/following{/other_user}","gists_url":"https://api.github.com/users/aodn/gists{/gist_id}","starred_url":"https://api.github.com/users/aodn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aodn/subscriptions","organizations_url":"https://api.github.com/users/aodn/orgs","repos_url":"https://api.github.com/users/aodn/repos","events_url":"https://api.github.com/users/aodn/events{/privacy}","received_events_url":"https://api.github.com/users/aodn/received_events","type":"Organization","site_admin":false},"repo":{"id":109786059,"name":"python-aodndata","full_name":"aodn/python-aodndata","owner":{"login":"aodn","id":2802166,"avatar_url":"https://avatars2.githubusercontent.com/u/2802166?v=4","gravatar_id":"","url":"https://api.github.com/users/aodn","html_url":"https://github.com/aodn","followers_url":"https://api.github.com/users/aodn/followers","following_url":"https://api.github.com/users/aodn/following{/other_user}","gists_url":"https://api.github.com/users/aodn/gists{/gist_id}","starred_url":"https://api.github.com/users/aodn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aodn/subscriptions","organizations_url":"https://api.github.com/users/aodn/orgs","repos_url":"https://api.github.com/users/aodn/repos","events_url":"https://api.github.com/users/aodn/events{/privacy}","received_events_url":"https://api.github.com/users/aodn/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/aodn/python-aodndata","description":null,"fork":false,"url":"https://api.github.com/repos/aodn/python-aodndata","forks_url":"https://api.github.com/repos/aodn/python-aodndata/forks","keys_url":"https://api.github.com/repos/aodn/python-aodndata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aodn/python-aodndata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aodn/python-aodndata/teams","hooks_url":"https://api.github.com/repos/aodn/python-aodndata/hooks","issue_events_url":"https://api.github.com/repos/aodn/python-aodndata/issues/events{/number}","events_url":"https://api.github.com/repos/aodn/python-aodndata/events","assignees_url":"https://api.github.com/repos/aodn/python-aodndata/assignees{/user}","branches_url":"https://api.github.com/repos/aodn/python-aodndata/branches{/branch}","tags_url":"https://api.github.com/repos/aodn/python-aodndata/tags","blobs_url":"https://api.github.com/repos/aodn/python-aodndata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aodn/python-aodndata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aodn/python-aodndata/git/refs{/sha}","trees_url":"https://api.github.com/repos/aodn/python-aodndata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aodn/python-aodndata/statuses/{sha}","languages_url":"https://api.github.com/repos/aodn/python-aodndata/languages","stargazers_url":"https://api.github.com/repos/aodn/python-aodndata/stargazers","contributors_url":"https://api.github.com/repos/aodn/python-aodndata/contributors","subscribers_url":"https://api.github.com/repos/aodn/python-aodndata/subscribers","subscription_url":"https://api.github.com/repos/aodn/python-aodndata/subscription","commits_url":"https://api.github.com/repos/aodn/python-aodndata/commits{/sha}","git_commits_url":"https://api.github.com/repos/aodn/python-aodndata/git/commits{/sha}","comments_url":"https://api.github.com/repos/aodn/python-aodndata/comments{/number}","issue_comment_url":"https://api.github.com/repos/aodn/python-aodndata/issues/comments{/number}","contents_url":"https://api.github.com/repos/aodn/python-aodndata/contents/{+path}","compare_url":"https://api.github.com/repos/aodn/python-aodndata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aodn/python-aodndata/merges","archive_url":"https://api.github.com/repos/aodn/python-aodndata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aodn/python-aodndata/downloads","issues_url":"https://api.github.com/repos/aodn/python-aodndata/issues{/number}","pulls_url":"https://api.github.com/repos/aodn/python-aodndata/pulls{/number}","milestones_url":"https://api.github.com/repos/aodn/python-aodndata/milestones{/number}","notifications_url":"https://api.github.com/repos/aodn/python-aodndata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aodn/python-aodndata/labels{/name}","releases_url":"https://api.github.com/repos/aodn/python-aodndata/releases{/id}","deployments_url":"https://api.github.com/repos/aodn/python-aodndata/deployments","created_at":"2017-11-07T04:17:30Z","updated_at":"2018-05-17T22:15:19Z","pushed_at":"2018-05-17T22:15:21Z","git_url":"git://github.com/aodn/python-aodndata.git","ssh_url":"git@github.com:aodn/python-aodndata.git","clone_url":"https://github.com/aodn/python-aodndata.git","svn_url":"https://github.com/aodn/python-aodndata","homepage":null,"size":237323,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":3,"license":null,"forks":0,"open_issues":3,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/32"},"html":{"href":"https://github.com/aodn/python-aodndata/pull/32"},"issue":{"href":"https://api.github.com/repos/aodn/python-aodndata/issues/32"},"comments":{"href":"https://api.github.com/repos/aodn/python-aodndata/issues/32/comments"},"review_comments":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/32/comments"},"review_comment":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/aodn/python-aodndata/pulls/32/commits"},"statuses":{"href":"https://api.github.com/repos/aodn/python-aodndata/statuses/eb0970e1e6615b3b0cfb2a85b4c679e9bc668519"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"lbesnard"}}